From a9fefd7f533cf9bdb6369d7159ab0df3e01b357d Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Wed, 6 May 2020 14:34:55 -0500 Subject: [PATCH] HBASE-24297 release scripts should be able to use a custom git repo * adds a optional -r [repo] arg * if the passed repo is on the local filesystem, creates a container mount * when cloning a local repo configure git to share objects with the local repo instead of copying * when cloning a local repo in a container configure the clone to have a remote that will work back on the host. closes #1725 Signed-off-by: stack Signed-off-by: Nick Dimiduk --- .../create-release/do-release-docker.sh | 60 ++++++++++++++++++- dev-support/create-release/release-build.sh | 14 ++--- dev-support/create-release/release-util.sh | 37 ++++++++++++ 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index fade6290629..568ada7507b 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -53,6 +53,7 @@ export PROJECT="${PROJECT:-hbase}" SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" +ORIG_PWD="$(pwd)" function usage { local NAME @@ -71,6 +72,7 @@ Options: -j [path] path to local JDK installation to use building. By default the script will use openjdk8 installed in the docker image. -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var + -r [repo] git repo to use for remote git operations. defaults to ASF gitbox for project. -s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release. If none specified, runs tag, then publish-dist, and then publish-release. 'publish-snapshot' is also an allowed, less used, option. @@ -82,13 +84,15 @@ WORKDIR= IMGTAG=latest JAVA= RELEASE_STEP= -while getopts "d:fhj:p:s:t:" opt; do +GIT_REPO= +while getopts "d:fhj:p:r:s:t:" opt; do case $opt in d) WORKDIR="$OPTARG" ;; f) DRY_RUN=0 ;; t) IMGTAG="$OPTARG" ;; j) JAVA="$OPTARG" ;; p) PROJECT="$OPTARG" ;; + r) GIT_REPO="$OPTARG" ;; s) RELEASE_STEP="$OPTARG" ;; h) usage ;; ?) error "Invalid option. Run with -h for help." ;; @@ -169,11 +173,65 @@ if [ -n "$JAVA" ]; then JAVA_VOL=(--volume "$JAVA:/opt/hbase-java") fi +#TODO some debug output would be good here +GIT_REPO_MOUNT=() +if [ -n "${GIT_REPO}" ]; then + case "${GIT_REPO}" in + # skip the easy to identify remote protocols + ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;; + # for sure local + /*) + GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo") + echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}" + GIT_REPO="/opt/hbase-repo" + ;; + # on the host but normally git wouldn't use the local optimization + file://*) + echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local." + GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo") + echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}" + GIT_REPO="/opt/hbase-repo" + ;; + # have to decide if it's a local path or the "scp-ish" remote + *) + declare colon_remove_prefix; + declare slash_remove_prefix; + declare local_path; + colon_remove_prefix="${GIT_REPO#*:}" + slash_remove_prefix="${GIT_REPO#*/}" + if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then + # if there was no colon at all, we assume this must be a local path + local_path="no colon at all" + elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then + # if there was a colon and there is no slash, then we assume it must be scp-style host + # and a relative path + + if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then + # Given the substrings made by removing everything up to the first colon and slash + # we can determine which comes first based on the longer substring length. + # if the slash is first, then we assume the colon is part of a path name and if the colon + # is first then it is the seperator between a scp-style host name and the path. + local_path="slash happened before a colon" + fi + fi + if [ -n "${local_path}" ]; then + # convert to an absolute path + GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")" + GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo") + echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}" + GIT_REPO="/opt/hbase-repo" + fi + ;; + esac + echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}" +fi + echo "Building $RELEASE_TAG; output will be at $WORKDIR/output" cmd=(docker run -ti \ --env-file "$ENVFILE" \ --volume "$WORKDIR:/opt/hbase-rm" \ "${JAVA_VOL[@]}" \ + "${GIT_REPO_MOUNT[@]}" \ "hbase-rm:$IMGTAG") echo "${cmd[*]}" "${cmd[@]}" diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index ef79d65d320..86cca652daa 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -111,12 +111,11 @@ if [[ "$1" == "tag" ]]; then set -o pipefail set -x # detailed logging during action check_get_passwords ASF_PASSWORD - check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \ - GIT_EMAIL GIT_NAME GIT_BRANCH - ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" - encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")" - encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")" - git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}" + check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH + if [ -z "${GIT_REPO}" ]; then + check_needed_vars ASF_USERNAME ASF_PASSWORD + fi + git_clone_overwrite # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' @@ -186,8 +185,7 @@ fi if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then ln -s "${PROJECT}.tag" "${PROJECT}" else - ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}" - git clone "$ASF_REPO" "${PROJECT}" + git_clone_overwrite fi cd "${PROJECT}" git checkout "$GIT_REF" diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 703fb854f69..2297ee7a9c1 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -398,6 +398,43 @@ function configure_maven { EOF } +# clone of the repo, deleting anything that exists in the working directory named after the project. +# optionally with auth details for pushing. +function git_clone_overwrite { + local asf_repo + if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then + error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'" + fi + rm -rf "${PROJECT}" + + if [[ -z "${GIT_REPO}" ]]; then + asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git" + echo "[INFO] clone will be of the gitbox repo for ${PROJECT}." + if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then + # Ugly! + encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") + encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") + GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}" + else + GIT_REPO="https://${asf_repo}" + fi + else + echo "[INFO] clone will be of provided git repo." + fi + # N.B. we use the shared flag because the clone is short lived and if a local repo repo was + # given this will let us refer to objects there directly instead of hardlinks or copying. + # The option is silently ignored for non-local repositories. see the note on git help clone + # for the --shared option for details. + git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}" + # If this was a host local git repo then add in an alternates and remote that will + # work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag + # for more info see 'git help remote' and 'git help repository-layout'. + if [ -n "$HOST_GIT_REPO" ]; then + echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates" + (cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}") + fi +} + # Writes report into cwd! function generate_api_report { local project="$1"