From 51991c4907262716e522e327edf96bea1fd3c494 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Thu, 17 Jun 2021 22:44:44 +0530 Subject: [PATCH] HADOOP-17724. Add Dockerfile for Debian 10 (#3038) --- dev-support/docker/Dockerfile_debian_10 | 107 ++++++++++++++++++ dev-support/docker/pkg-resolver/packages.json | 64 +++++++++++ .../docker/pkg-resolver/platforms.json | 3 +- 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 dev-support/docker/Dockerfile_debian_10 diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 new file mode 100644 index 00000000000..8b7cbde7e62 --- /dev/null +++ b/dev-support/docker/Dockerfile_debian_10 @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM debian:10 + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +##### +# For installing the latest packages +##### +RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME /usr +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 + +####### +# Set env vars for SpotBugs 4.2.2 +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.7.1 +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx3072m + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION true + +#### +# Install packages +#### +RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh debian:10 +RUN pkg-resolver/install-boost.sh debian:10 +RUN pkg-resolver/install-protobuf.sh debian:10 +RUN pkg-resolver/install-hadolint.sh debian:10 +RUN pkg-resolver/install-intel-isa-l.sh debian:10 + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 4ebbc900fe4..cb4b7914d54 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,30 +1,39 @@ { "ant": { + "debian:10": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "centos:7": "ant", "centos:8": "ant" }, "apt-utils": { + "debian:10": "apt-utils", "ubuntu:focal": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { + "debian:10": "automake", "ubuntu:focal": "automake", "ubuntu:focal::arch64": "automake", "centos:7": "automake", "centos:8": "automake" }, "bats": { + "debian:10": "bats", "ubuntu:focal": "bats", "ubuntu:focal::arch64": "bats" }, "build-essential": { + "debian:10": "build-essential", "ubuntu:focal": "build-essential", "ubuntu:focal::arch64": "build-essential", "centos:7": "build-essential" }, "bzip2": { + "debian:10": [ + "bzip2", + "libbz2-dev" + ], "ubuntu:focal": [ "bzip2", "libbz2-dev" @@ -43,16 +52,22 @@ ] }, "clang": { + "debian:10": "clang", "ubuntu:focal": "clang", "ubuntu:focal::arch64": "clang", "centos:7": "clang", "centos:8": "clang" }, "cmake": { + "debian:10": "cmake", "ubuntu:focal": "cmake", "ubuntu:focal::arch64": "cmake" }, "curl": { + "debian:10": [ + "curl", + "libcurl4-openssl-dev" + ], "ubuntu:focal": [ "curl", "libcurl4-openssl-dev" @@ -71,6 +86,7 @@ ] }, "doxygen": { + "debian:10": "doxygen", "ubuntu:focal": "doxygen", "ubuntu:focal::arch64": "doxygen", "centos:7": "doxygen" @@ -79,6 +95,10 @@ "centos:8": "dnf" }, "fuse": { + "debian:10": [ + "fuse", + "libfuse-dev" + ], "ubuntu:focal": [ "fuse", "libfuse-dev" @@ -99,6 +119,10 @@ ] }, "gcc": { + "debian:10": [ + "gcc", + "g++" + ], "ubuntu:focal": [ "gcc", "g++" @@ -113,36 +137,46 @@ ] }, "git": { + "debian:10": "git", "ubuntu:focal": "git", "ubuntu:focal::arch64": "git", "centos:7": "git", "centos:8": "git" }, "gnupg-agent": { + "debian:10": "gnupg-agent", "ubuntu:focal": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { + "debian:10": "hugo", "ubuntu:focal": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { + "debian:10": "libbcprov-java", "ubuntu:focal": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { + "debian:10": "libtool", "ubuntu:focal": "libtool", "ubuntu:focal::arch64": "libtool", "centos:7": "libtool", "centos:8": "libtool" }, "openssl": { + "debian:10": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", "centos:7": "openssl-devel", "centos:8": "openssl-devel" }, "protocol-buffers": { + "debian:10": [ + "libprotobuf-dev", + "libprotoc-dev" + ], "ubuntu:focal": [ "libprotobuf-dev", "libprotoc-dev" @@ -153,17 +187,23 @@ ] }, "sasl": { + "debian:10": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", "centos:7": "cyrus-sasl-devel", "centos:8": "cyrus-sasl-devel" }, "snappy": { + "debian:10": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", "ubuntu:focal::arch64": "libsnappy-dev", "centos:7": "snappy-devel" }, "zlib": { + "debian:10": [ + "libzstd-dev", + "zlib1g-dev" + ], "ubuntu:focal": [ "libzstd-dev", "zlib1g-dev" @@ -182,6 +222,7 @@ ] }, "locales": { + "debian:10": "locales", "ubuntu:focal": "locales", "ubuntu:focal::arch64": "locales" }, @@ -193,16 +234,23 @@ "centos:7": "libpmem-devel" }, "make": { + "debian:10": "make", "ubuntu:focal": "make", "ubuntu:focal::arch64": "make", "centos:7": "make", "centos:8": "make" }, "maven": { + "debian:10": "maven", "ubuntu:focal": "maven", "ubuntu:focal::arch64": "maven" }, "javascript": { + "debian:10": [ + "nodejs", + "node-yarn", + "npm" + ], "ubuntu:focal": [ "nodejs", "node-yarn", @@ -216,6 +264,7 @@ ] }, "java": { + "debian:10": "openjdk-11-jdk", "ubuntu:focal": [ "openjdk-8-jdk", "openjdk-11-jdk" @@ -226,17 +275,26 @@ ] }, "pinentry-curses": { + "debian:10": "pinentry-curses", "ubuntu:focal": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", "centos:7": "pinentry-curses", "centos:8": "pinentry-curses" }, "pkg-config": { + "debian:10": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:focal::arch64": "pkg-config", "centos:8": "pkg-config" }, "python": { + "debian:10": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], "ubuntu:focal": [ "python3", "python3-pip", @@ -266,12 +324,14 @@ ] }, "rsync": { + "debian:10": "rsync", "ubuntu:focal": "rsync", "ubuntu:focal::arch64": "rsync", "centos:7": "rsync", "centos:8": "rsync" }, "shellcheck": { + "debian:10": "shellcheck", "ubuntu:focal": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, @@ -280,22 +340,26 @@ "centos:8": "perl-Digest-SHA" }, "software-properties-common": { + "debian:10": "software-properties-common", "ubuntu:focal": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { + "debian:10": "sudo", "ubuntu:focal": "sudo", "ubuntu:focal::arch64": "sudo", "centos:7": "sudo", "centos:8": "sudo" }, "valgrind": { + "debian:10": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:focal::arch64": "valgrind", "centos:7": "valgrind", "centos:8": "valgrind" }, "yasm": { + "debian:10": "yasm", "ubuntu:focal": "yasm", "ubuntu:focal::arch64": "yasm" } diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 4861c5dd55c..93e2a93df42 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -2,5 +2,6 @@ "ubuntu:focal", "ubuntu:focal::arch64", "centos:7", - "centos:8" + "centos:8", + "debian:10" ] \ No newline at end of file