From 1c0b2edde93f8cf2724b3e20bcc40a5443597a22 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Tue, 8 Jun 2021 08:41:08 +0530 Subject: [PATCH] HADOOP-17727. Modularize docker images (#3043) Reviewed-by: Inigo Goiri --- dev-support/bin/create-release | 2 +- dev-support/docker/Dockerfile | 149 ++------- dev-support/docker/Dockerfile_aarch64 | 119 ++----- dev-support/docker/Dockerfile_centos_7 | 151 ++------- dev-support/docker/Dockerfile_centos_8 | 144 ++------- dev-support/docker/README.md | 74 +++++ .../docker/pkg-resolver/check_platform.py | 50 +++ .../docker/pkg-resolver/install-boost.sh | 56 ++++ .../docker/pkg-resolver/install-cmake.sh | 53 +++ .../pkg-resolver/install-common-pkgs.sh | 28 ++ .../docker/pkg-resolver/install-epel.sh | 49 +++ .../docker/pkg-resolver/install-hadolint.sh | 35 ++ .../pkg-resolver/install-intel-isa-l.sh | 58 ++++ .../docker/pkg-resolver/install-maven.sh | 49 +++ .../docker/pkg-resolver/install-nodejs.sh | 54 ++++ .../docker/pkg-resolver/install-protobuf.sh | 57 ++++ .../docker/pkg-resolver/install-spotbugs.sh | 50 +++ .../docker/pkg-resolver/install-yasm.sh | 49 +++ .../docker/pkg-resolver/install-zstandard.sh | 53 +++ dev-support/docker/pkg-resolver/packages.json | 302 ++++++++++++++++++ .../docker/pkg-resolver/platforms.json | 6 + dev-support/docker/pkg-resolver/resolve.py | 61 ++++ 22 files changed, 1190 insertions(+), 459 deletions(-) create mode 100644 dev-support/docker/README.md create mode 100644 dev-support/docker/pkg-resolver/check_platform.py create mode 100644 dev-support/docker/pkg-resolver/install-boost.sh create mode 100644 dev-support/docker/pkg-resolver/install-cmake.sh create mode 100644 dev-support/docker/pkg-resolver/install-common-pkgs.sh create mode 100644 dev-support/docker/pkg-resolver/install-epel.sh create mode 100644 dev-support/docker/pkg-resolver/install-hadolint.sh create mode 100644 dev-support/docker/pkg-resolver/install-intel-isa-l.sh create mode 100644 dev-support/docker/pkg-resolver/install-maven.sh create mode 100644 dev-support/docker/pkg-resolver/install-nodejs.sh create mode 100644 dev-support/docker/pkg-resolver/install-protobuf.sh create mode 100644 dev-support/docker/pkg-resolver/install-spotbugs.sh create mode 100644 dev-support/docker/pkg-resolver/install-yasm.sh create mode 100644 dev-support/docker/pkg-resolver/install-zstandard.sh create mode 100644 dev-support/docker/pkg-resolver/packages.json create mode 100644 dev-support/docker/pkg-resolver/platforms.json create mode 100644 dev-support/docker/pkg-resolver/resolve.py diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release index 39a5d0d3198..31ae6ee1b06 100755 --- a/dev-support/bin/create-release +++ b/dev-support/bin/create-release @@ -514,7 +514,7 @@ function dockermode echo "USER ${user_name}" printf "\n\n" - ) | docker build -t "${imgname}" - + ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/ run docker run -i -t \ --privileged \ diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index ef2f1562613..fac364bbd43 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,56 +32,20 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -# hadolint ignore=DL3008 +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - openjdk-8-jdk \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -97,89 +61,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - -### -# Install hadolint -#### -RUN curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint \ - && chmod a+rx /bin/hadolint \ - && shasum -a 512 /bin/hadolint | \ - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -RUN mkdir -p /opt/isa-l-src \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends automake yasm \ - && apt-get clean \ - && curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz \ - && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ - && cd /opt/isa-l-src \ - && ./autogen.sh \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/isa-l-src - ### # Avoid out of memory errors in builds ### @@ -188,6 +79,16 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m # Skip gpg verification when downloading Yetus via yetus-wrapper ENV HADOOP_SKIP_YETUS_VERIFICATION true +#### +# Install packages +#### +RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh ubuntu:focal +RUN pkg-resolver/install-boost.sh ubuntu:focal +RUN pkg-resolver/install-protobuf.sh ubuntu:focal +RUN pkg-resolver/install-hadolint.sh ubuntu:focal +RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 5c2613268a6..dd0348961f4 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true ###### -# Install common dependencies from packages. Versions here are either -# sufficient or irrelevant. +# Platform package dependency resolver ###### -# hadolint ignore=DL3008 +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - openjdk-8-jdk \ - phantomjs \ - pinentry-curses \ - pkg-config \ - python2.7 \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -103,59 +61,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - ### # Avoid out of memory errors in builds ### @@ -167,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true # Force PhantomJS to be in 'headless' mode, do not connect to Xwindow ENV QT_QPA_PLATFORM offscreen +#### +# Install packages +#### +RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64 +RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64 +RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64 + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 319942e2f6d..c74230e758c 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -23,79 +23,28 @@ WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y centos-release-scl \ - && yum install -y devtoolset-9 \ && yum install -y \ - ant \ - build-essential \ - bzip2 \ - bzip2-devel \ - clang \ - curl \ - cyrus-sasl-devel \ - doxygen \ - fuse \ - fuse-libs \ - fuse-devel \ - git \ - libcurl-devel \ - libtirpc-devel \ - libpmem-devel \ - libtool \ - lz4-devel \ - make \ - openssl-devel \ - pinentry-curses \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rsync \ - snappy-devel \ - sudo \ - valgrind \ - zlib-devel + centos-release-scl \ + python3 \ + && yum install -y $(pkg-resolver/resolve.py centos:7) # Set GCC 9 as the default C/C++ compiler RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -#### -# Install Maven 3.6.3 -#### -RUN mkdir -p /opt/maven /tmp/maven \ - && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \ - && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven - -#### -# Install CMake 3.19 -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/cmake /opt/cmake \ - && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \ - && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \ - && cd /opt/cmake || exit && ./bootstrap \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -#### -# Install zstandard -#### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/zstd /tmp/zstd \ - && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \ - && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \ - && cd /opt/zstd || exit \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 +# TODO: Set locale ###### # Set env vars required to build Hadoop @@ -106,68 +55,26 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin" ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install Node.js -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/node \ - && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \ - && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \ - && cd /tmp/node || exit \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:7 +RUN pkg-resolver/install-cmake.sh centos:7 +RUN pkg-resolver/install-zstandard.sh centos:7 +RUN pkg-resolver/install-yasm.sh centos:7 +RUN pkg-resolver/install-protobuf.sh centos:7 +RUN pkg-resolver/install-boost.sh centos:7 +RUN pkg-resolver/install-spotbugs.sh centos:7 +RUN pkg-resolver/install-nodejs.sh centos:7 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 7f1f6b4ac22..b4109d825e4 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -23,48 +23,30 @@ WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y \ - ant \ - bzip2 \ - bzip2-devel \ - clang \ - curl \ - cyrus-sasl-devel \ - dnf \ - fuse \ - git \ - libcurl-devel \ - fuse \ - fuse-libs \ - fuse-devel \ - libtool \ - libtirpc-devel \ - lz4-devel \ - make \ - openssl-devel \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rsync \ - sudo \ - valgrind \ - zlib-devel + && yum install -y python3 \ + && yum install -y $(pkg-resolver/resolve.py centos:8) #### # Install EPEL #### -RUN mkdir -p /tmp/epel \ - && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ - -o /tmp/epel/epel-release-latest-8.noarch.rpm \ - && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm +RUN pkg-resolver/install-epel.sh centos:8 RUN dnf --enablerepo=powertools install -y \ doxygen \ - snappy-devel + snappy-devel \ + yasm RUN dnf install -y \ bouncycastle \ @@ -78,100 +60,34 @@ RUN dnf install -y \ RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -#### -# Install Maven 3.6.3 -#### -RUN mkdir -p /opt/maven /tmp/maven \ - && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \ - && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven - -#### -# Install CMake 3.19 -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/cmake /opt/cmake \ - && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \ - && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \ - && cd /opt/cmake || exit && ./bootstrap \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -#### -# Install zstandard -#### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/zstd /tmp/zstd \ - && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \ - && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \ - && cd /opt/zstd || exit \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 +# TODO: Set locale ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 +ENV MAVEN_HOME /opt/maven ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 +# Set env vars for Google Protobuf ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:8 +RUN pkg-resolver/install-cmake.sh centos:8 +RUN pkg-resolver/install-boost.sh centos:8 +RUN pkg-resolver/install-spotbugs.sh centos:8 +RUN pkg-resolver/install-protobuf.sh centos:8 +RUN pkg-resolver/install-zstandard.sh centos:8 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/README.md b/dev-support/docker/README.md new file mode 100644 index 00000000000..32269c0929a --- /dev/null +++ b/dev-support/docker/README.md @@ -0,0 +1,74 @@ + + +# Docker images for building Hadoop + +This folder contains the Dockerfiles for building Hadoop on various platforms. + +# Dependency management + +The mode of installation of the dependencies needed for building Hadoop varies from one platform to +the other. Different platforms have different toolchains. Some packages tend to be polymorphic +across platforms and most commonly, a package that's readily available in one platform's toolchain +isn't available on another. We thus, resort to building and installing the package from source, +causing duplication of code since this needs to be done for all the Dockerfiles pertaining to all +the platforms. We need a system to track a dependency - for a package - for a platform. Thus, +there's a lot of diversity that needs to be handled for managing package dependencies and +`pkg-resolver` caters to that. + +## Supported platforms + +`pkg-resolver/platforms.json` contains a list of the supported platforms for dependency management. + +## Package dependencies + +`pkg-resolver/packages.json` maps a dependency to a given platform. Here's the schema of this JSON. + +```json +{ + "dependency_1": { + "platform_1": "package_1", + "platform_2": [ + "package_1", + "package_2" + ] + }, + "dependency_2": { + "platform_1": [ + "package_1", + "package_2", + "package_3" + ] + } +} +``` + +The root JSON element contains unique _dependency_ children. This in turn contains the name of the _ +platforms_ and the list of _packages_ to be installed for that platform. Just to give an example of +how to interpret the above JSON - + +1. For `dependency_1`, `package_1` needs to be installed for `platform_1`. +2. For `dependency_2`, `package_1` and `package_2` needs to be installed for `platform_2`. +3. For `dependency_2`, `package_1`, `package_3` and `package_3` needs to be installed for + `platform_1`. + +## Standalone packages + +Most commonly, some packages are not available across the toolchains in various platforms. Thus, we +would need to build and install them. Since we need to do this across all the Dockerfiles for all +the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put +the build steps in a `pkg-resolver/install-.sh` and invoke this in all the Dockerfiles. \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/check_platform.py b/dev-support/docker/pkg-resolver/check_platform.py new file mode 100644 index 00000000000..fa5529a58be --- /dev/null +++ b/dev-support/docker/pkg-resolver/check_platform.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks whether the given platform is supported for building Apache Hadoop +""" + +import json +import sys + + +def get_platforms(): + """ + :return: A list of the supported platforms managed by pkg-resolver. + """ + + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: + return json.loads(platforms_file.read()) + + +def is_supported_platform(platform): + """ + :param platform: The name of the platform + :return: Whether the platform is supported + """ + return platform in get_platforms() + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print('ERROR: Expecting 1 argument, {} were provided'.format(len(sys.argv) - 1), + file=sys.stderr) + sys.exit(1) + + sys.exit(0 if is_supported_platform(sys.argv[1]) else 1) diff --git a/dev-support/docker/pkg-resolver/install-boost.sh b/dev-support/docker/pkg-resolver/install-boost.sh new file mode 100644 index 00000000000..eaca09effa2 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-boost.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.72.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.72.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.72.0" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/boost-library && + curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && + mv boost_1_72_0.tar.bz2 /opt/boost-library && + cd /opt/boost-library && + tar --bzip2 -xf boost_1_72_0.tar.bz2 && + cd /opt/boost-library/boost_1_72_0 && + ./bootstrap.sh --prefix=/usr/ && + ./b2 --without-python install && + cd /root && + rm -rf /opt/boost-library +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-cmake.sh b/dev-support/docker/pkg-resolver/install-cmake.sh new file mode 100644 index 00000000000..29e2733e701 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-cmake.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.19.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.19.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.19.0" ]; then + # hadolint ignore=DL3003 + mkdir -p /tmp/cmake /opt/cmake && + curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz && + tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake && + cd /opt/cmake || exit && ./bootstrap && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh new file mode 100644 index 00000000000..d6756eea73d --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +###### +# Install pylint and python-dateutil +###### +pip3 install pylint==2.6.0 python-dateutil==2.8.1 + +###### +# Install bower +###### +# hadolint ignore=DL3008 +npm install -g bower@1.8.8 diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh new file mode 100644 index 00000000000..875dce3a9ae --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-epel.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="8" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "8" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "8" ]; then + mkdir -p /tmp/epel && + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + -o /tmp/epel/epel-release-latest-8.noarch.rpm && + rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-hadolint.sh b/dev-support/docker/pkg-resolver/install-hadolint.sh new file mode 100644 index 00000000000..1e2081f38c4 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-hadolint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && + chmod a+rx /bin/hadolint && + shasum -a 512 /bin/hadolint | + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' diff --git a/dev-support/docker/pkg-resolver/install-intel-isa-l.sh b/dev-support/docker/pkg-resolver/install-intel-isa-l.sh new file mode 100644 index 00000000000..c6b4de78228 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-intel-isa-l.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="2.29.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "2.29.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "2.29.0" ]; then + # hadolint ignore=DL3003,DL3008 + mkdir -p /opt/isa-l-src && + curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz && + tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && + cd /opt/isa-l-src && + ./autogen.sh && + ./configure && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/isa-l-src +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-maven.sh b/dev-support/docker/pkg-resolver/install-maven.sh new file mode 100644 index 00000000000..f9ff961a190 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-maven.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.6.3" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.6.3" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.6.3" ]; then + mkdir -p /opt/maven /tmp/maven && + curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ + -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz && + tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-nodejs.sh b/dev-support/docker/pkg-resolver/install-nodejs.sh new file mode 100644 index 00000000000..5ba1c228086 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-nodejs.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="14.16.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "14.16.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "14.16.1" ]; then + # hadolint ignore=DL3003 + mkdir -p /tmp/node && + curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz && + tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node && + cd /tmp/node || exit && + ./configure && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-protobuf.sh b/dev-support/docker/pkg-resolver/install-protobuf.sh new file mode 100644 index 00000000000..7303b404822 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-protobuf.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.7.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.7.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.7.1" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/protobuf-src && + curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz && + tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + cd /opt/protobuf-src && + ./configure --prefix=/opt/protobuf && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/protobuf-src +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-spotbugs.sh b/dev-support/docker/pkg-resolver/install-spotbugs.sh new file mode 100644 index 00000000000..65a8f2e6924 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-spotbugs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="4.2.2" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "4.2.2" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "4.2.2" ]; then + mkdir -p /opt/spotbugs && + curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz && + tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && + chmod +x /opt/spotbugs/bin/* +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-yasm.sh b/dev-support/docker/pkg-resolver/install-yasm.sh new file mode 100644 index 00000000000..a5f6162bc38 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-yasm.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.2.0-4" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.2.0-4" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.2.0-4" ]; then + mkdir -p /tmp/yasm && + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ + -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm && + rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-zstandard.sh b/dev-support/docker/pkg-resolver/install-zstandard.sh new file mode 100644 index 00000000000..3aafd469d2b --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-zstandard.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.4.9" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.4.9" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.4.9" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/zstd /tmp/zstd && + curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz && + tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd && + cd /opt/zstd || exit && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json new file mode 100644 index 00000000000..4ebbc900fe4 --- /dev/null +++ b/dev-support/docker/pkg-resolver/packages.json @@ -0,0 +1,302 @@ +{ + "ant": { + "ubuntu:focal": "ant", + "ubuntu:focal::arch64": "ant", + "centos:7": "ant", + "centos:8": "ant" + }, + "apt-utils": { + "ubuntu:focal": "apt-utils", + "ubuntu:focal::arch64": "apt-utils" + }, + "automake": { + "ubuntu:focal": "automake", + "ubuntu:focal::arch64": "automake", + "centos:7": "automake", + "centos:8": "automake" + }, + "bats": { + "ubuntu:focal": "bats", + "ubuntu:focal::arch64": "bats" + }, + "build-essential": { + "ubuntu:focal": "build-essential", + "ubuntu:focal::arch64": "build-essential", + "centos:7": "build-essential" + }, + "bzip2": { + "ubuntu:focal": [ + "bzip2", + "libbz2-dev" + ], + "ubuntu:focal::arch64": [ + "bzip2", + "libbz2-dev" + ], + "centos:7": [ + "bzip2", + "bzip2-devel" + ], + "centos:8": [ + "bzip2", + "bzip2-devel" + ] + }, + "clang": { + "ubuntu:focal": "clang", + "ubuntu:focal::arch64": "clang", + "centos:7": "clang", + "centos:8": "clang" + }, + "cmake": { + "ubuntu:focal": "cmake", + "ubuntu:focal::arch64": "cmake" + }, + "curl": { + "ubuntu:focal": [ + "curl", + "libcurl4-openssl-dev" + ], + "ubuntu:focal::arch64": [ + "curl", + "libcurl4-openssl-dev" + ], + "centos:7": [ + "curl", + "libcurl-devel" + ], + "centos:8": [ + "curl", + "libcurl-devel" + ] + }, + "doxygen": { + "ubuntu:focal": "doxygen", + "ubuntu:focal::arch64": "doxygen", + "centos:7": "doxygen" + }, + "dnf": { + "centos:8": "dnf" + }, + "fuse": { + "ubuntu:focal": [ + "fuse", + "libfuse-dev" + ], + "ubuntu:focal::arch64": [ + "fuse", + "libfuse-dev" + ], + "centos:7": [ + "fuse", + "fuse-libs", + "fuse-devel" + ], + "centos:8": [ + "fuse", + "fuse-libs", + "fuse-devel" + ] + }, + "gcc": { + "ubuntu:focal": [ + "gcc", + "g++" + ], + "ubuntu:focal::arch64": [ + "gcc", + "g++" + ], + "centos:7": [ + "centos-release-scl", + "devtoolset-9" + ] + }, + "git": { + "ubuntu:focal": "git", + "ubuntu:focal::arch64": "git", + "centos:7": "git", + "centos:8": "git" + }, + "gnupg-agent": { + "ubuntu:focal": "gnupg-agent", + "ubuntu:focal::arch64": "gnupg-agent" + }, + "hugo": { + "ubuntu:focal": "hugo", + "ubuntu:focal::arch64": "hugo" + }, + "libbcprov-java": { + "ubuntu:focal": "libbcprov-java", + "ubuntu:focal::arch64": "libbcprov-java" + }, + "libtool": { + "ubuntu:focal": "libtool", + "ubuntu:focal::arch64": "libtool", + "centos:7": "libtool", + "centos:8": "libtool" + }, + "openssl": { + "ubuntu:focal": "libssl-dev", + "ubuntu:focal::arch64": "libssl-dev", + "centos:7": "openssl-devel", + "centos:8": "openssl-devel" + }, + "protocol-buffers": { + "ubuntu:focal": [ + "libprotobuf-dev", + "libprotoc-dev" + ], + "ubuntu:focal::arch64": [ + "libprotobuf-dev", + "libprotoc-dev" + ] + }, + "sasl": { + "ubuntu:focal": "libsasl2-dev", + "ubuntu:focal::arch64": "libsasl2-dev", + "centos:7": "cyrus-sasl-devel", + "centos:8": "cyrus-sasl-devel" + }, + "snappy": { + "ubuntu:focal": "libsnappy-dev", + "ubuntu:focal::arch64": "libsnappy-dev", + "centos:7": "snappy-devel" + }, + "zlib": { + "ubuntu:focal": [ + "libzstd-dev", + "zlib1g-dev" + ], + "ubuntu:focal::arch64": [ + "libzstd-dev", + "zlib1g-dev" + ], + "centos:7": [ + "zlib-devel", + "lz4-devel" + ], + "centos:8": [ + "zlib-devel", + "lz4-devel" + ] + }, + "locales": { + "ubuntu:focal": "locales", + "ubuntu:focal::arch64": "locales" + }, + "libtirpc-devel": { + "centos:7": "libtirpc-devel", + "centos:8": "libtirpc-devel" + }, + "libpmem": { + "centos:7": "libpmem-devel" + }, + "make": { + "ubuntu:focal": "make", + "ubuntu:focal::arch64": "make", + "centos:7": "make", + "centos:8": "make" + }, + "maven": { + "ubuntu:focal": "maven", + "ubuntu:focal::arch64": "maven" + }, + "javascript": { + "ubuntu:focal": [ + "nodejs", + "node-yarn", + "npm" + ], + "ubuntu:focal::arch64": [ + "nodejs", + "node-yarn", + "npm", + "phantomjs" + ] + }, + "java": { + "ubuntu:focal": [ + "openjdk-8-jdk", + "openjdk-11-jdk" + ], + "ubuntu:focal::arch64": [ + "openjdk-8-jdk", + "openjdk-11-jdk" + ] + }, + "pinentry-curses": { + "ubuntu:focal": "pinentry-curses", + "ubuntu:focal::arch64": "pinentry-curses", + "centos:7": "pinentry-curses", + "centos:8": "pinentry-curses" + }, + "pkg-config": { + "ubuntu:focal": "pkg-config", + "ubuntu:focal::arch64": "pkg-config", + "centos:8": "pkg-config" + }, + "python": { + "ubuntu:focal": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "ubuntu:focal::arch64": [ + "python2.7", + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "centos:7": [ + "python3", + "python3-pip", + "python3-setuptools", + "python3-wheel" + ], + "centos:8": [ + "python3", + "python3-pip", + "python3-setuptools", + "python3-wheel" + ] + }, + "rsync": { + "ubuntu:focal": "rsync", + "ubuntu:focal::arch64": "rsync", + "centos:7": "rsync", + "centos:8": "rsync" + }, + "shellcheck": { + "ubuntu:focal": "shellcheck", + "ubuntu:focal::arch64": "shellcheck" + }, + "shasum": { + "centos:7": "perl-Digest-SHA", + "centos:8": "perl-Digest-SHA" + }, + "software-properties-common": { + "ubuntu:focal": "software-properties-common", + "ubuntu:focal::arch64": "software-properties-common" + }, + "sudo": { + "ubuntu:focal": "sudo", + "ubuntu:focal::arch64": "sudo", + "centos:7": "sudo", + "centos:8": "sudo" + }, + "valgrind": { + "ubuntu:focal": "valgrind", + "ubuntu:focal::arch64": "valgrind", + "centos:7": "valgrind", + "centos:8": "valgrind" + }, + "yasm": { + "ubuntu:focal": "yasm", + "ubuntu:focal::arch64": "yasm" + } +} \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json new file mode 100644 index 00000000000..4861c5dd55c --- /dev/null +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -0,0 +1,6 @@ +[ + "ubuntu:focal", + "ubuntu:focal::arch64", + "centos:7", + "centos:8" +] \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py new file mode 100644 index 00000000000..5d62edd323e --- /dev/null +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Platform package dependency resolver for building Apache Hadoop. +""" + +import json +import sys +from check_platform import is_supported_platform + + +def get_packages(platform): + """ + Resolve and get the list of packages to install for the given platform. + + :param platform: The platform for which the packages needs to be resolved. + :return: A list of resolved packages to install. + """ + with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file: + pkgs = json.loads(pkg_file.read()) + packages = [] + for platforms in filter(lambda x: x.get(platform) is not None, pkgs.values()): + if isinstance(platforms.get(platform), list): + packages.extend(platforms.get(platform)) + else: + packages.append(platforms.get(platform)) + return packages + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), + file=sys.stderr) + sys.exit(1) + + platform_arg = sys.argv[1] + if not is_supported_platform(platform_arg): + print( + 'ERROR: The given platform {} is not supported. ' + 'Please refer to platforms.json for a list of supported platforms'.format( + platform_arg), file=sys.stderr) + sys.exit(1) + + packages_to_install = get_packages(platform_arg) + print(' '.join(packages_to_install))