HADOOP-16797. Add Dockerfile for ARM builds. Contributed by Vinayakumar B. (#1801)

2020-01-13 10:40:29 +05:30 · 2020-01-13 10:40:29 +05:30 · 52b360a928
parent d842dfffa5
commit 52b360a928
4 changed files with 265 additions and 3 deletions
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@ -204,6 +204,11 @@ function set_defaults
  DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile"
  DOCKERRAN=false

+  CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+  if [ "$CPU_ARCH" = "aarch64" ]; then
+    DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
+  fi
+
  # Extract Java version from ${BASEDIR}/pom.xml
  # doing this outside of maven means we can do this before
  # the docker container comes up...
@ -249,7 +254,9 @@ function startgpgagent
      eval $("${GPGAGENT}" --daemon \
        --options "${LOGDIR}/gpgagent.conf" \
        --log-file="${LOGDIR}/create-release-gpgagent.log")
-      GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
+      GPGAGENTPID=$(pgrep "${GPGAGENT}")
+      GPG_AGENT_INFO="$HOME/.gnupg/S.gpg-agent:$GPGAGENTPID:1"
+      export GPG_AGENT_INFO
    fi

    if [[ -n "${GPG_AGENT_INFO}" ]]; then
@ -499,7 +506,12 @@ function dockermode

    # we always force build with the OpenJDK JDK
    # but with the correct version
-    echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
+    if [ "$CPU_ARCH" = "aarch64" ]; then
+      echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
+    else
+      echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
+    fi
+
    echo "USER ${user_name}"
    printf "\n\n"
  ) | docker build -t "${imgname}" -
--- a/dev-support/docker/Dockerfile_aarch64
+++ b/dev-support/docker/Dockerfile_aarch64
@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM ubuntu:xenial
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+######
+# Install common dependencies from packages. Versions here are either
+# sufficient or irrelevant.
+#
+# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
+# Ubuntu Java.  See Java section below!
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends \
+        apt-utils \
+        build-essential \
+        bzip2 \
+        clang \
+        curl \
+        doxygen \
+        fuse \
+        g++ \
+        gcc \
+        git \
+        gnupg-agent \
+        libbz2-dev \
+        libcurl4-openssl-dev \
+        libfuse-dev \
+        libprotobuf-dev \
+        libprotoc-dev \
+        libsasl2-dev \
+        libsnappy-dev \
+        libssl-dev \
+        libtool \
+        libzstd1-dev \
+        locales \
+        make \
+        pinentry-curses \
+        pkg-config \
+        python \
+        python2.7 \
+        python-pip \
+        python-pkg-resources \
+        python-setuptools \
+        python-wheel \
+        rsync \
+        software-properties-common \
+        snappy \
+        sudo \
+        valgrind \
+        zlib1g-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+#######
+# OpenJDK 8
+#######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+######
+# Install cmake 3.1.0 (3.5.1 ships with Xenial)
+# There is no cmake binary available for aarch64. Build from source.
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/cmake/src \
+    && curl -L -s -S \
+      https://cmake.org/files/v3.1/cmake-3.1.0-1-src.tar.bz2 \
+      -o /opt/cmake/cmake-src.tar.bz2 \
+    && tar xvjf /opt/cmake/cmake-src.tar.bz2 -C /opt/cmake/src \
+    && cd /opt/cmake/src \
+    && tar xvjf cmake-3.1.0.tar.bz2 \
+    && cd cmake-3.1.0 && patch -p0 -i ../cmake-3.1.0-1.patch && mkdir .build && cd .build \
+    && ../bootstrap --parallel=2 \
+    && make -j2 && ./bin/cpack \
+    && tar xzf cmake-3.1.0-Linux-aarch64.tar.gz --strip-components 1 -C /opt/cmake \
+    && cd /opt/cmake && rm -rf /opt/cmake/src
+ENV CMAKE_HOME /opt/cmake
+ENV PATH "${PATH}:/opt/cmake/bin"
+
+######
+# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/protobuf-src \
+    && curl -L -s -S \
+      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
+      -o /opt/protobuf.tar.gz \
+    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
+    && cd /opt/protobuf-src \
+    && ./configure --prefix=/opt/protobuf \
+    && make install \
+    && cd /root \
+    && rm -rf /opt/protobuf-src
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+######
+# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends maven \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+ENV MAVEN_HOME /usr
+
+######
+# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
+# Ant is needed for findbugs
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends findbugs ant \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+ENV FINDBUGS_HOME /usr
+
+####
+# Install shellcheck (0.4.6, the latest as of 2017-09-26)
+####
+# hadolint ignore=DL3008
+RUN add-apt-repository -y ppa:hvr/ghc \
+    && apt-get -q update \
+    && apt-get -q install -y --no-install-recommends shellcheck \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+####
+# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
+####
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends bats \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+####
+# Install pylint at fixed version (2.0.0 removed python2 support)
+# https://github.com/PyCQA/pylint/issues/2294
+####
+RUN pip2 install pylint==1.9.2
+
+####
+# Install dateutil.parser
+####
+RUN pip2 install python-dateutil==2.7.3
+
+###
+# Install node.js 8.17.0 for web UI framework (4.2.6 ships with Xenial)
+###
+RUN curl -L -s -S https://deb.nodesource.com/setup_8.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs=8.17.0-1nodesource1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install -g bower@1.8.8
+
+###
+## Install Yarn 1.12.1 for web UI framework
+####
+RUN curl -s -S https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list \
+    && apt-get -q update \
+    && apt-get install -y --no-install-recommends yarn=1.21.1-1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+###
+# Install phantomjs built for aarch64
+####
+RUN mkdir -p /opt/phantomjs \
+  && curl -L -s -S \
+    https://github.com/liusheng/phantomjs/releases/download/2.1.1/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
+   -o /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
+  && tar xvjf /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 --strip-components 1 -C /opt/phantomjs \
+  && cp /opt/phantomjs/bin/phantomjs /usr/bin/ \
+  && rm -rf /opt/phantomjs
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx1536m
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Hugo static website generator (for new hadoop site docs)
+RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-ARM64.deb \
+    && dpkg --install hugo.deb \
+    && rm hugo.deb
+
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@ -142,6 +142,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
      <artifactId>findbugs</artifactId>
      <version>3.0.1</version>
      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>xml-apis</groupId>
+          <artifactId>xml-apis</artifactId>
+        </exclusion>
+      </exclusions>
    </dependency>

    <dependency>
--- a/start-build-env.sh
+++ b/start-build-env.sh
@ -19,7 +19,15 @@ set -e               # exit on error

 cd "$(dirname "$0")" # connect to root

-docker build -t hadoop-build dev-support/docker
+DOCKER_DIR=dev-support/docker
+DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
+
+CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+if [ "$CPU_ARCH" = "aarch64" ]; then
+  DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
+fi
+
+docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR

 USER_NAME=${SUDO_USER:=$USER}
 USER_ID=$(id -u "${USER_NAME}")
@ -80,5 +88,6 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
  -v "${PWD}:/home/${USER_NAME}/hadoop${V_OPTS:-}" \
  -w "/home/${USER_NAME}/hadoop" \
  -v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
+  -v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg${V_OPTS:-}" \
  -u "${USER_NAME}" \
  "hadoop-build-${USER_ID}" "$@"