HBASE-24297 release scripts should be able to use a custom git repo
* adds a optional -r [repo] arg * if the passed repo is on the local filesystem, creates a container mount * when cloning a local repo configure git to share objects with the local repo instead of copying * when cloning a local repo in a container configure the clone to have a remote that will work back on the host. closes #1725 Signed-off-by: stack <stack@apache.org> Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>
This commit is contained in:
Sean Busbey
parent
8ff8e70edf
commit
a9fefd7f53
|
|
@ -53,6 +53,7 @@ export PROJECT="${PROJECT:-hbase}"
|
|||
SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=SCRIPTDIR/release-util.sh
|
||||
. "$SELF/release-util.sh"
|
||||
ORIG_PWD="$(pwd)"
|
||||
|
||||
function usage {
|
||||
local NAME
|
||||
|
|
@ -71,6 +72,7 @@ Options:
|
|||
-j [path] path to local JDK installation to use building. By default the script will
|
||||
use openjdk8 installed in the docker image.
|
||||
-p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
|
||||
-r [repo] git repo to use for remote git operations. defaults to ASF gitbox for project.
|
||||
-s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
|
||||
If none specified, runs tag, then publish-dist, and then publish-release.
|
||||
'publish-snapshot' is also an allowed, less used, option.
|
||||
|
|
@ -82,13 +84,15 @@ WORKDIR=
|
|||
IMGTAG=latest
|
||||
JAVA=
|
||||
RELEASE_STEP=
|
||||
while getopts "d:fhj:p:s:t:" opt; do
|
||||
GIT_REPO=
|
||||
while getopts "d:fhj:p:r:s:t:" opt; do
|
||||
case $opt in
|
||||
d) WORKDIR="$OPTARG" ;;
|
||||
f) DRY_RUN=0 ;;
|
||||
t) IMGTAG="$OPTARG" ;;
|
||||
j) JAVA="$OPTARG" ;;
|
||||
p) PROJECT="$OPTARG" ;;
|
||||
r) GIT_REPO="$OPTARG" ;;
|
||||
s) RELEASE_STEP="$OPTARG" ;;
|
||||
h) usage ;;
|
||||
?) error "Invalid option. Run with -h for help." ;;
|
||||
|
|
@ -169,11 +173,65 @@ if [ -n "$JAVA" ]; then
|
|||
JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
|
||||
fi
|
||||
|
||||
#TODO some debug output would be good here
|
||||
GIT_REPO_MOUNT=()
|
||||
if [ -n "${GIT_REPO}" ]; then
|
||||
case "${GIT_REPO}" in
|
||||
# skip the easy to identify remote protocols
|
||||
ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
|
||||
# for sure local
|
||||
/*)
|
||||
GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
|
||||
echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
|
||||
GIT_REPO="/opt/hbase-repo"
|
||||
;;
|
||||
# on the host but normally git wouldn't use the local optimization
|
||||
file://*)
|
||||
echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local."
|
||||
GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo")
|
||||
echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
|
||||
GIT_REPO="/opt/hbase-repo"
|
||||
;;
|
||||
# have to decide if it's a local path or the "scp-ish" remote
|
||||
*)
|
||||
declare colon_remove_prefix;
|
||||
declare slash_remove_prefix;
|
||||
declare local_path;
|
||||
colon_remove_prefix="${GIT_REPO#*:}"
|
||||
slash_remove_prefix="${GIT_REPO#*/}"
|
||||
if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
|
||||
# if there was no colon at all, we assume this must be a local path
|
||||
local_path="no colon at all"
|
||||
elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
|
||||
# if there was a colon and there is no slash, then we assume it must be scp-style host
|
||||
# and a relative path
|
||||
|
||||
if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
|
||||
# Given the substrings made by removing everything up to the first colon and slash
|
||||
# we can determine which comes first based on the longer substring length.
|
||||
# if the slash is first, then we assume the colon is part of a path name and if the colon
|
||||
# is first then it is the seperator between a scp-style host name and the path.
|
||||
local_path="slash happened before a colon"
|
||||
fi
|
||||
fi
|
||||
if [ -n "${local_path}" ]; then
|
||||
# convert to an absolute path
|
||||
GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
|
||||
GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
|
||||
echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
|
||||
GIT_REPO="/opt/hbase-repo"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
|
||||
fi
|
||||
|
||||
echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
|
||||
cmd=(docker run -ti \
|
||||
--env-file "$ENVFILE" \
|
||||
--volume "$WORKDIR:/opt/hbase-rm" \
|
||||
"${JAVA_VOL[@]}" \
|
||||
"${GIT_REPO_MOUNT[@]}" \
|
||||
"hbase-rm:$IMGTAG")
|
||||
echo "${cmd[*]}"
|
||||
"${cmd[@]}"
|
||||
|
|
|
|||
|
|
@ -111,12 +111,11 @@ if [[ "$1" == "tag" ]]; then
|
|||
set -o pipefail
|
||||
set -x # detailed logging during action
|
||||
check_get_passwords ASF_PASSWORD
|
||||
check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \
|
||||
GIT_EMAIL GIT_NAME GIT_BRANCH
|
||||
ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git"
|
||||
encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")"
|
||||
encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")"
|
||||
git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}"
|
||||
check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
|
||||
if [ -z "${GIT_REPO}" ]; then
|
||||
check_needed_vars ASF_USERNAME ASF_PASSWORD
|
||||
fi
|
||||
git_clone_overwrite
|
||||
|
||||
# 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
|
||||
# $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
|
||||
|
|
@ -186,8 +185,7 @@ fi
|
|||
if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
|
||||
ln -s "${PROJECT}.tag" "${PROJECT}"
|
||||
else
|
||||
ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}"
|
||||
git clone "$ASF_REPO" "${PROJECT}"
|
||||
git_clone_overwrite
|
||||
fi
|
||||
cd "${PROJECT}"
|
||||
git checkout "$GIT_REF"
|
||||
|
|
|
|||
|
|
@ -398,6 +398,43 @@ function configure_maven {
|
|||
EOF
|
||||
}
|
||||
|
||||
# clone of the repo, deleting anything that exists in the working directory named after the project.
|
||||
# optionally with auth details for pushing.
|
||||
function git_clone_overwrite {
|
||||
local asf_repo
|
||||
if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then
|
||||
error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'"
|
||||
fi
|
||||
rm -rf "${PROJECT}"
|
||||
|
||||
if [[ -z "${GIT_REPO}" ]]; then
|
||||
asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git"
|
||||
echo "[INFO] clone will be of the gitbox repo for ${PROJECT}."
|
||||
if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
|
||||
# Ugly!
|
||||
encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")
|
||||
encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")
|
||||
GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
|
||||
else
|
||||
GIT_REPO="https://${asf_repo}"
|
||||
fi
|
||||
else
|
||||
echo "[INFO] clone will be of provided git repo."
|
||||
fi
|
||||
# N.B. we use the shared flag because the clone is short lived and if a local repo repo was
|
||||
# given this will let us refer to objects there directly instead of hardlinks or copying.
|
||||
# The option is silently ignored for non-local repositories. see the note on git help clone
|
||||
# for the --shared option for details.
|
||||
git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}"
|
||||
# If this was a host local git repo then add in an alternates and remote that will
|
||||
# work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag
|
||||
# for more info see 'git help remote' and 'git help repository-layout'.
|
||||
if [ -n "$HOST_GIT_REPO" ]; then
|
||||
echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates"
|
||||
(cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}")
|
||||
fi
|
||||
}
|
||||
|
||||
# Writes report into cwd!
|
||||
function generate_api_report {
|
||||
local project="$1"
|
||||
|
|
|
|||
Loading…
Reference in New Issue