HADOOP-13245. Fix up some misc create-release issues (aw)

(cherry picked from commit e2f640942b)

dev-support/bin/create-release

@@ -240,6 +240,8 @@ function set_defaults
   OSNAME=$(uname -s)
 
   PUBKEYFILE="https://dist.apache.org/repos/dist/release/hadoop/common/KEYS"
+
+  SIGN=false
 }
 
 function startgpgagent
@@ -247,11 +249,23 @@ function startgpgagent
   if [[ "${SIGN}" = true ]]; then
     if [[ -n "${GPGAGENT}" && -z "${GPG_AGENT_INFO}" ]]; then
       echo "starting gpg agent"
-      touch "${LOGDIR}/gpgagent.conf"
+      echo "default-cache-ttl 7200" > "${LOGDIR}/gpgagent.conf"
+      # shellcheck disable=2046
       eval $("${GPGAGENT}" --daemon \
         --options "${LOGDIR}/gpgagent.conf" \
-        --log-file=${LOGDIR}/create-release-gpgagent.log)
+        --log-file="${LOGDIR}/create-release-gpgagent.log")
-      GPGAGENTPID=$(echo ${GPG_AGENT_INFO} | cut -f 2 -d:)
+      GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
+    fi
+
+    if [[ -n "${GPG_AGENT_INFO}" ]]; then
+      echo "Warming the gpg-agent cache prior to calling maven"
+      # warm the agent's cache:
+      touch "${LOGDIR}/warm"
+      ${GPG} --use-agent --armor --output "${LOGDIR}/warm.asc" --detach-sig "${LOGDIR}/warm"
+      rm "${LOGDIR}/warm.asc" "${LOGDIR}/warm"
+    else
+      SIGN=false
+      hadoop_error "ERROR: Unable to launch or acquire gpg-agent. Disable signing."
     fi
   fi
 }
@@ -259,7 +273,7 @@ function startgpgagent
 function stopgpgagent
 {
   if [[ -n "${GPGAGENTPID}" ]]; then
-    kill ${GPGAGENTPID}
+    kill "${GPGAGENTPID}"
   fi
 }
 
@@ -273,7 +287,7 @@ function usage
   echo "--mvncache=[path]       Path to the maven cache to use"
   echo "--native                Also build the native components"
   echo "--rc-label=[label]      Add this label to the builds"
-  echo "--sign                  Use .gnupg dir to sign the jars"
+  echo "--sign                  Use .gnupg dir to sign the artifacts and jars"
   echo "--version=[version]     Use an alternative version string"
 }
 
@@ -330,6 +344,16 @@ function option_parse
     SIGN=false
   fi
 
+  if [[ "${SIGN}" = true ]]; then
+    if [[ -n "${GPG_AGENT_INFO}" ]]; then
+      echo "NOTE: Using existing gpg-agent. If the default-cache-ttl"
+      echo "is set to less than ~20 mins, maven commands will fail."
+    elif [[ -z "${GPGAGENT}" ]]; then
+      hadoop_error "ERROR: No gpg-agent. Disabling signing capability."
+      SIGN=false
+    fi
+  fi
+
   DOCKERCMD=$(command -v docker)
   if [[ "${DOCKER}" = true && -z "${DOCKERCMD}" ]]; then
       hadoop_error "ERROR: docker binary not found. Disabling docker mode."
@@ -439,6 +463,11 @@ function dockermode
     # make sure we put some space between, just in case last
     # line isn't an empty line or whatever
     printf "\n\n"
+
+    # force a new image for every run to make it easier to remove later
+    echo "LABEL org.apache.hadoop.create-release=\"cr-${RANDOM}\""
+
+    # setup ownerships, etc
     echo "RUN groupadd --non-unique -g ${group_id} ${user_name}"
     echo "RUN useradd -g ${group_id} -u ${user_id} -m ${user_name}"
     echo "RUN chown -R ${user_name} /home/${user_name}"
@@ -490,19 +519,27 @@ function makearelease
 
   big_console_header "Maven Build and Install"
 
+  if [[ "${SIGN}" = true ]]; then
+    signflags=("-Psign" "-Dgpg.useagent=true" -Dgpg.executable="${GPG}")
+  fi
+
   # Create SRC and BIN tarballs for release,
-  # Using 'install’ goal instead of 'package' so artifacts are available
-  # in the Maven local cache for the site generation
-  #
   # shellcheck disable=SC2046
   run_and_redirect "${LOGDIR}/mvn_install.log" \
-    "${MVN}" "${MVN_ARGS[@]}" install -Pdist,src \
+    "${MVN}" "${MVN_ARGS[@]}" install \
+      -Pdist,src \
+      "${signflags[@]}" \
       -DskipTests -Dtar $(hadoop_native_flags)
 
-  big_console_header "Maven Site"
-
   # Create site for release
-  run_and_redirect "${LOGDIR}/mvn_site.log" "${MVN}" "${MVN_ARGS[@]}" site site:stage -Pdist,src,releasedocs
+  # we need to do install again so that jdiff and
+  # a few other things get registered in the maven
+  # universe correctly
+  run_and_redirect "${LOGDIR}/mvn_site.log" \
+    "${MVN}" "${MVN_ARGS[@]}" install \
+      site site:stage \
+      -DskipTests \
+      -Pdist,src,releasedocs,docs
 
   big_console_header "Staging the release"
 
@@ -561,16 +598,16 @@ function signartifacts
   big_console_header "Signing the release"
 
   for i in ${ARTIFACTS_DIR}/*; do
-    gpg --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
+    ${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
-    gpg --print-mds "${i}" > "${i}.mds"
+    ${GPG} --print-mds "${i}" > "${i}.mds"
     domd5 "${i}"
   done
 
   if [[ "${ASFRELEASE}" = true ]]; then
     echo "Fetching the Apache Hadoop KEYS file..."
     curl -L "${PUBKEYFILE}" -o "${BASEDIR}/target/KEYS"
-    gpg --import --trustdb "${BASEDIR}/target/testkeysdb" "${BASEDIR}/target/KEYS"
+    ${GPG} --import --trustdb "${BASEDIR}/target/testkeysdb" "${BASEDIR}/target/KEYS"
-    gpg --verify --trustdb "${BASEDIR}/target/testkeysdb" \
+    ${GPG} --verify --trustdb "${BASEDIR}/target/testkeysdb" \
       "${ARTIFACTS_DIR}/hadoop-${HADOOP_VERSION}${RC_LABEL}.tar.gz.asc" \
         "${ARTIFACTS_DIR}/hadoop-${HADOOP_VERSION}${RC_LABEL}.tar.gz"
     if [[ $? != 0 ]]; then

dev-support/docker/Dockerfile

@@ -95,6 +95,11 @@ ENV FINDBUGS_HOME /opt/findbugs
 RUN apt-get install -y cabal-install
 RUN cabal update && cabal install shellcheck --global
 
+####
+# Install dateutil.parser
+####
+RUN pip install python-dateutil
+
 ###
 # Avoid out of memory errors in builds
 ###

hadoop-common-project/hadoop-common/pom.xml

@@ -821,11 +821,12 @@
                     </goals>
                     <configuration>
                         <executable>${basedir}/../../dev-support/bin/releasedocmaker</executable>
-                        <workingDirectory>src/site/markdown/release/</workingDirectory>
                         <requiresOnline>true</requiresOnline>
                         <arguments>
                             <argument>--index</argument>
                             <argument>--license</argument>
+                            <argument>--outputdir</argument>
+                            <argument>${basedir}/src/site/markdown/release</argument>
                             <argument>--project</argument>
                             <argument>HADOOP</argument>
                             <argument>--project</argument>

pom.xml

@@ -99,6 +99,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <maven-dependency-plugin.version>2.8</maven-dependency-plugin.version>
     <maven-enforcer-plugin.version>1.3.1</maven-enforcer-plugin.version>
     <maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
+    <maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
     <apache-rat-plugin.version>0.10</apache-rat-plugin.version>
     <wagon-ssh.version>1.0</wagon-ssh.version>
     <maven-clover2-plugin.version>3.3.0</maven-clover2-plugin.version>
@@ -499,6 +500,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-gpg-plugin</artifactId>
+            <version>${maven-gpg-plugin.version}</version>
             <executions>
               <execution>
                 <id>sign-artifacts</id>