HADOOP-18401. No ARM binaries in branch-3.3.x releases. (#4953)

Fix the branch-3.3 docker image and create-release scripts to work on arm 64 and macbook m1

Contributed by Ayush Saxena and Steve Loughran

dev-support/bin/create-release

@@ -205,7 +205,8 @@ function set_defaults
   DOCKERRAN=false
 
   CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
-  if [ "$CPU_ARCH" = "aarch64" ]; then
+  if [[ "$CPU_ARCH" = "aarch64"  || "$CPU_ARCH" = "arm64" ]]; then
+    echo "Using aarch64 docker file"
     DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
   fi
 
@@ -513,7 +514,7 @@ function dockermode
 
     # we always force build with the OpenJDK JDK
     # but with the correct version
-    if [ "$CPU_ARCH" = "aarch64" ]; then
+    if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
       echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
     else
       echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"

dev-support/docker/Dockerfile_aarch64

@@ -17,7 +17,7 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-FROM ubuntu:bionic
+FROM ubuntu:focal
 
 WORKDIR /root
 
@@ -53,6 +53,7 @@ RUN apt-get -q update \
         gcc \
         git \
         gnupg-agent \
+        hugo \
         libbcprov-java \
         libbz2-dev \
         libcurl4-openssl-dev \
@@ -63,10 +64,13 @@ RUN apt-get -q update \
         libsnappy-dev \
         libssl-dev \
         libtool \
-        libzstd1-dev \
+        libzstd-dev \
         locales \
         make \
         maven \
+        nodejs \
+        node-yarn \
+        npm \
         openjdk-11-jdk \
         openjdk-8-jdk \
         phantomjs \
@@ -74,7 +78,6 @@ RUN apt-get -q update \
         pkg-config \
         python2.7 \
         python3 \
-        python3-dev \
         python3-pip \
         python3-pkg-resources \
         python3-setuptools \
@@ -110,7 +113,7 @@ RUN mkdir -p /opt/spotbugs \
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 ######
-# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic)
+# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
 ######
 # hadolint ignore=DL3003
 RUN mkdir -p /opt/protobuf-src \
@@ -132,6 +135,12 @@ ENV PATH "${PATH}:/opt/protobuf/bin"
 ####
 RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
 
+####
+# Install bower
+####
+# hadolint ignore=DL3008
+RUN npm install -g bower@1.8.8
+
 ###
 # Avoid out of memory errors in builds
 ###
@@ -149,12 +158,6 @@ ENV QT_QPA_PLATFORM offscreen
 # YETUS CUT HERE
 ###
 
-# Hugo static website generator (for new hadoop site docs)
-RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-ARM64.deb \
-    && dpkg --install hugo.deb \
-    && rm hugo.deb
-
-
 # Add a welcome message and environment checks.
 COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
 RUN chmod 755 /root/hadoop_env_checks.sh

hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch

@@ -1,5 +1,5 @@
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-index a277abd6e13..1d131d5db6e 100644
+index fef8c4b7e4b..8d2b9339706 100644
 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
 @@ -42,18 +42,6 @@
@@ -14,7 +14,7 @@ index a277abd6e13..1d131d5db6e 100644
 -   *              the annotations of the source object.)
 -   * @param desc  the description of the source (or null. See above.)
 -   * @return the source object
--   * @exception MetricsException
+-   * @exception MetricsException Metrics Exception.
 -   */
 -  public abstract <T> T register(String name, String desc, T source);
 -
@@ -38,7 +38,7 @@ index a277abd6e13..1d131d5db6e 100644
 +   *              the annotations of the source object.)
 +   * @param desc  the description of the source (or null. See above.)
 +   * @return the source object
-    * @exception MetricsException
+    * @exception MetricsException Metrics Exception.
     */
 -  public abstract <T extends MetricsSink>
 -  T register(String name, String desc, T sink);