HBASE-24297 release scripts should be able to use a custom git repo

* adds a optional -r [repo] arg
* if the passed repo is on the local filesystem, creates a container mount
* when cloning a local repo configure git to share objects with the local repo instead of copying
* when cloning a local repo in a container configure the clone to have a remote that will work back on the host.

closes #1725

Signed-off-by: stack <stack@apache.org>
Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>

dev-support/create-release/do-release-docker.sh

@@ -53,6 +53,7 @@ export PROJECT="${PROJECT:-hbase}"
 SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=SCRIPTDIR/release-util.sh
 . "$SELF/release-util.sh"
+ORIG_PWD="$(pwd)"
 
 function usage {
   local NAME
@@ -71,6 +72,7 @@ Options:
   -j [path]    path to local JDK installation to use building. By default the script will
                use openjdk8 installed in the docker image.
   -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
+  -r [repo]    git repo to use for remote git operations. defaults to ASF gitbox for project.
   -s [step]    runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
                If none specified, runs tag, then publish-dist, and then publish-release.
                'publish-snapshot' is also an allowed, less used, option.
@@ -82,13 +84,15 @@ WORKDIR=
 IMGTAG=latest
 JAVA=
 RELEASE_STEP=
-while getopts "d:fhj:p:s:t:" opt; do
+GIT_REPO=
+while getopts "d:fhj:p:r:s:t:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
     f) DRY_RUN=0 ;;
     t) IMGTAG="$OPTARG" ;;
     j) JAVA="$OPTARG" ;;
     p) PROJECT="$OPTARG" ;;
+    r) GIT_REPO="$OPTARG" ;;
     s) RELEASE_STEP="$OPTARG" ;;
     h) usage ;;
     ?) error "Invalid option. Run with -h for help." ;;
@@ -169,11 +173,65 @@ if [ -n "$JAVA" ]; then
   JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
 fi
 
+#TODO some debug output would be good here
+GIT_REPO_MOUNT=()
+if [ -n "${GIT_REPO}" ]; then
+  case "${GIT_REPO}" in
+    # skip the easy to identify remote protocols
+    ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
+    # for sure local
+    /*)
+      GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+      echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+      GIT_REPO="/opt/hbase-repo"
+      ;;
+    # on the host but normally git wouldn't use the local optimization
+    file://*)
+      echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local."
+      GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo")
+      echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+      GIT_REPO="/opt/hbase-repo"
+      ;;
+    # have to decide if it's a local path or the "scp-ish" remote
+    *)
+      declare colon_remove_prefix;
+      declare slash_remove_prefix;
+      declare local_path;
+      colon_remove_prefix="${GIT_REPO#*:}"
+      slash_remove_prefix="${GIT_REPO#*/}"
+      if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
+        # if there was no colon at all, we assume this must be a local path
+        local_path="no colon at all"
+      elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
+        # if there was a colon and there is no slash, then we assume it must be scp-style host
+        # and a relative path
+
+        if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
+          # Given the substrings made by removing everything up to the first colon and slash
+          # we can determine which comes first based on the longer substring length.
+          # if the slash is first, then we assume the colon is part of a path name and if the colon
+          # is first then it is the seperator between a scp-style host name and the path.
+          local_path="slash happened before a colon"
+        fi
+      fi
+      if [ -n "${local_path}" ]; then
+        # convert to an absolute path
+        GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
+        GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+        echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+        GIT_REPO="/opt/hbase-repo"
+      fi
+      ;;
+  esac
+  echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+fi
+
 echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
 cmd=(docker run -ti \
   --env-file "$ENVFILE" \
   --volume "$WORKDIR:/opt/hbase-rm" \
   "${JAVA_VOL[@]}" \
+  "${GIT_REPO_MOUNT[@]}" \
   "hbase-rm:$IMGTAG")
 echo "${cmd[*]}"
 "${cmd[@]}"

dev-support/create-release/release-build.sh

@@ -111,12 +111,11 @@ if [[ "$1" == "tag" ]]; then
   set -o pipefail
   set -x  # detailed logging during action
   check_get_passwords ASF_PASSWORD
-  check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \
+  check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
-      GIT_EMAIL GIT_NAME GIT_BRANCH
+  if [ -z "${GIT_REPO}" ]; then
-  ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git"
+    check_needed_vars ASF_USERNAME ASF_PASSWORD
-  encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")"
+  fi
-  encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")"
+  git_clone_overwrite
-  git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}"
 
   # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
   # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
@@ -186,8 +185,7 @@ fi
 if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
   ln -s "${PROJECT}.tag" "${PROJECT}"
 else
-  ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}"
+  git_clone_overwrite
-  git clone "$ASF_REPO" "${PROJECT}"
 fi
 cd "${PROJECT}"
 git checkout "$GIT_REF"

dev-support/create-release/release-util.sh

@@ -398,6 +398,43 @@ function configure_maven {
 EOF
 }
 
+# clone of the repo, deleting anything that exists in the working directory named after the project.
+# optionally with auth details for pushing.
+function git_clone_overwrite {
+  local asf_repo
+  if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then
+    error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'"
+  fi
+  rm -rf "${PROJECT}"
+
+  if [[ -z "${GIT_REPO}" ]]; then
+    asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git"
+    echo "[INFO] clone will be of the gitbox repo for ${PROJECT}."
+    if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
+      # Ugly!
+      encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")
+      encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")
+      GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
+    else
+      GIT_REPO="https://${asf_repo}"
+    fi
+  else
+    echo "[INFO] clone will be of provided git repo."
+  fi
+  # N.B. we use the shared flag because the clone is short lived and if a local repo repo was
+  #      given this will let us refer to objects there directly instead of hardlinks or copying.
+  #      The option is silently ignored for non-local repositories. see the note on git help clone
+  #      for the --shared option for details.
+  git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}"
+  # If this was a host local git repo then add in an alternates and remote that will
+  # work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag
+  # for more info see 'git help remote' and 'git help repository-layout'.
+  if [ -n "$HOST_GIT_REPO" ]; then
+    echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates"
+    (cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}")
+  fi
+}
+
 # Writes report into cwd!
 function generate_api_report {
   local project="$1"