HBASE-25354 Update create-release scripts; rotted

README
 Add note on ssh-agent.

dev-support/create-release/do-release.sh
 move gpg check to non-docker context. Also cleanup tmp files when done.

dev-support/create-release/hbase-rm/Dockerfile
dev-support/create-release/mac-sshd-gpg-agent/Dockerfile
 Hack to update packages... the old ones  no longer available.

dev-support/create-release/release-util.sh
 Allow that there are no JIRA changes in a release. Good for testing.

dev-support/create-release/README.txt

@@ -9,19 +9,21 @@ To run a build w/o invoking docker (not recommended!), use _do_release.sh_.
 
 Both scripts will query interactively for needed parameters and passphrases.
 For explanation of the parameters, execute:
+
  $ release-build.sh --help
 
-Before starting the RC build, run a reconciliation of what is in
+Before starting the RC build, run a reconciliation of what is in JIRA with
-JIRA with what is in the commit log. Make sure they align and that
+what is in the commit log. Make sure they align and that anomalies are
-anomalies are explained up in JIRA.
+explained up in JIRA.
 
 See http://hbase.apache.org/book.html#maven.release
 
-Regardless of where your release build will run (locally, locally in docker, on a remote machine,
+Regardless of where your release build will run (locally, locally in docker,
-etc) you will need a local gpg-agent with access to your secret keys. A quick way to tell gpg
+on a remote machine, etc) you will need a local gpg-agent with access to your
-to clear out state and start a gpg-agent is via the following command phrase:
+secret keys. A quick way to tell gpg to clear out state and start a gpg-agent
+is via the following command phrase:
 
-$ gpgconf --kill all && gpg-connect-agent /bye
+ $ gpgconf --kill all && gpg-connect-agent /bye
 
 Before starting an RC build, make sure your local gpg-agent has configs
 to properly handle your credentials, especially if you want to avoid
@@ -33,6 +35,8 @@ on caching the unlocked secret via ~/.gnupg/gpg-agent.conf
   default-cache-ttl 86400
   max-cache-ttl 86400
 
+Similarly, run ssh-agent with your ssh key added if building with docker.
+
 Running a build on GCE is easy enough. Here are some notes if of use.
 Create an instance. 4CPU/15G/10G disk seems to work well enough.
 Once up, run the below to make your machine fit for RC building:

dev-support/create-release/do-release-docker.sh

@@ -302,7 +302,7 @@ if [ "${HOST_OS}" == "DARWIN" ]; then
       > "${WORKDIR}/gpg-agent-proxy.known_hosts"
   if [ -s "${WORKDIR}/gpg-agent-proxy.known_hosts" ]; then
     echo "Your ssh known_hosts does not include the entries for the gpg-agent proxy container."
-    echo "The following entry(ies) arre missing:"
+    echo "The following entry(ies) are missing:"
     sed -e 's/^/    /' "${WORKDIR}/gpg-agent-proxy.known_hosts"
     read -r -p "Okay to add these entries to ${HOME}/.ssh/known_hosts? [y/n] " ANSWER
     if [ "$ANSWER" != "y" ]; then

dev-support/create-release/do-release.sh

@@ -17,6 +17,10 @@
 # limitations under the License.
 #
 
+# Make a tmp dir into which we put files cleaned-up on exit.
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+
 set -e
 # Use the adjacent do-release-docker.sh instead, if you can.
 # Otherwise, this runs core of the release creation.
@@ -84,19 +88,22 @@ if [ "$RUNNING_IN_DOCKER" = "1" ]; then
 else
   # Outside docker, need to ask for information about the release.
   get_release_info
+
+  # Run this stuff when not in docker to check gpg.
+  gpg_test_file="${TMPDIR}/gpg_test.$$.txt"
+  echo "Testing gpg signing ${GPG} ${GPG_ARGS[@]} --detach --armor --sign ${gpg_test_file}"
+  echo "foo" > "${gpg_test_file}"
+  if ! "${GPG}" "${GPG_ARGS[@]}" --detach --armor --sign "${gpg_test_file}" ; then
+    gpg_agent_help
+  fi
+  # In --batch mode we have to be explicit about what we are verifying
+  if ! "${GPG}" "${GPG_ARGS[@]}" --verify "${gpg_test_file}.asc" "${gpg_test_file}" ; then
+    gpg_agent_help
+  fi
 fi
 
 GPG_TTY="$(tty)"
 export GPG_TTY
-echo "Testing gpg signing."
-echo "foo" > gpg_test.txt
-if ! "${GPG}" "${GPG_ARGS[@]}" --detach --armor --sign gpg_test.txt ; then
-  gpg_agent_help
-fi
-# In --batch mode we have to be explicit about what we are verifying
-if ! "${GPG}" "${GPG_ARGS[@]}" --verify gpg_test.txt.asc gpg_test.txt ; then
-  gpg_agent_help
-fi
 
 if [[ -z "$RELEASE_STEP" ]]; then
   # If doing all stages, leave out 'publish-snapshot'

dev-support/create-release/hbase-rm/Dockerfile

@@ -34,7 +34,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
     libxml2-dev='2.9.4+dfsg1-*' \
     lsof='4.89+dfsg-*' \
     maven='3.6.0-*' \
-    openjdk-8-jdk='8u252-b09-*' \
+    openjdk-8-jdk='8u*' \
     python-pip='9.0.1-*' \
     subversion='1.9.7-*' \
     wget='1.19.4-*' \

dev-support/create-release/mac-sshd-gpg-agent/Dockerfile

@@ -83,7 +83,7 @@ FROM ubuntu:18.04
 # into the container rather than launching a new docker container.
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
   && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-  openssh-server=1:7.6p1-4ubuntu0.3 gnupg2=2.2.4-1ubuntu1.2 && mkdir /run/sshd \
+  openssh-server=1:7.6p1-4ubuntu0.3 gnupg2=2.2.4-1ubuntu1.3 && mkdir /run/sshd \
   && echo "StreamLocalBindUnlink yes" >> /etc/ssh/sshd_config \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*

dev-support/create-release/release-util.sh

@@ -501,10 +501,17 @@ function update_releasenotes {
   local jira_project
   local timing_token
   timing_token="$(start_step)"
+  changelog="CHANGELOG.${jira_fix_version}.md"
+  releasenotes="RELEASENOTES.${jira_fix_version}.md"
+  if [ -f ${changelog} ]; then
+    rm ${changelog}
+  fi
+  if [ -f ${releasenotes} ]; then
+    rm ${releasenotes}
+  fi
   jira_project="$(get_jira_name "$(basename "$project_dir")")"
   "${YETUS_HOME}/bin/releasedocmaker" -p "${jira_project}" --fileversions -v "${jira_fix_version}" \
-      -l --sortorder=newer --skip-credits
+      -l --sortorder=newer --skip-credits || true
-  pwd
   # First clear out the changes written by previous RCs.
   if [ -f "${project_dir}/CHANGES.md" ]; then
     sed -i -e \
@@ -517,24 +524,35 @@ function update_releasenotes {
         "${project_dir}/RELEASENOTES.md" || true
   fi
 
+  # Yetus will not generate CHANGES if no JIRAs fixed against the release version
+  # (Could happen if a release were bungled such that we had to make a new one
+  # without changes)
+  if [ ! -f "${changelog}" ]; then
+    echo -e "## Release ${jira_fix_version} - Unreleased (as of `date`)\nNo changes\n" > "${changelog}"
+  fi
+  if [ ! -f "${releasenotes}" ]; then
+    echo -e "# hbase ${jira_fix_version} Release Notes\nNo changes\n" > "${releasenotes}"
+  fi
+
   # The releasedocmaker call above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md.
   if [ -f "${project_dir}/CHANGES.md" ]; then
     # To insert into project's CHANGES.md...need to cut the top off the
     # CHANGELOG.X.X.X.md file removing license and first line and then
     # insert it after the license comment closing where we have a
     # DO NOT REMOVE marker text!
-    sed -i -e '/## Release/,$!d' "CHANGELOG.${jira_fix_version}.md"
+    sed -i -e '/## Release/,$!d' "${changelog}"
-    sed -i -e "/DO NOT REMOVE/r CHANGELOG.${jira_fix_version}.md" "${project_dir}/CHANGES.md"
+    sed -i -e '2,${/^# HBASE Changelog/d;}' "${project_dir}/CHANGES.md"
+    sed -i -e "/DO NOT REMOVE/r ${changelog}" "${project_dir}/CHANGES.md"
   else
-    mv "CHANGELOG.${jira_fix_version}.md" "${project_dir}/CHANGES.md"
+    mv "${changelog}" "${project_dir}/CHANGES.md"
   fi
   if [ -f "${project_dir}/RELEASENOTES.md" ]; then
     # Similar for RELEASENOTES but slightly different.
-    sed -i -e '/Release Notes/,$!d' "RELEASENOTES.${jira_fix_version}.md"
+    sed -i -e '/Release Notes/,$!d' "${releasenotes}"
-    sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${jira_fix_version}.md" \
+    sed -i -e '2,${/^# RELEASENOTES/d;}' "${project_dir}/RELEASENOTES.md"
-        "${project_dir}/RELEASENOTES.md"
+    sed -i -e "/DO NOT REMOVE/r ${releasenotes}" "${project_dir}/RELEASENOTES.md"
   else
-    mv "RELEASENOTES.${jira_fix_version}.md" "${project_dir}/RELEASENOTES.md"
+    mv "${releasenotes}" "${project_dir}/RELEASENOTES.md"
   fi
   stop_step "${timing_token}"
 }