diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile index 5cc322b46f6..4f1d238d775 100644 --- a/dev-support/create-release/hbase-rm/Dockerfile +++ b/dev-support/create-release/hbase-rm/Dockerfile @@ -21,7 +21,6 @@ # * Java 8 FROM ubuntu:22.04 - # Install extra needed repos and refresh. # # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch @@ -39,6 +38,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \ python3-pip='22.0.2+dfsg-*' \ subversion='1.14.1-*' \ wget='1.21.2-*' \ + patch='2.7.6-*' \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \ diff --git a/dev-support/create-release/prepend_releasenotes.py b/dev-support/create-release/prepend_releasenotes.py index b5bbec1274e..41be2c1c18a 100755 --- a/dev-support/create-release/prepend_releasenotes.py +++ b/dev-support/create-release/prepend_releasenotes.py @@ -31,6 +31,7 @@ with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', if pattern.match(line): break line = prev_r.readline() + w.writelines('# RELEASENOTES') for newline in new_r: w.writelines(newline) while line: diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index fa9844a2df0..f3d8798be46 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -130,11 +130,13 @@ if [[ "$1" == "tag" ]]; then # So, here we prepend the project name to the version, but only for the hbase sub-projects. jira_fix_version="${RELEASE_VERSION}" shopt -s nocasematch - if [[ "${PROJECT}" =~ ^hbase- ]]; then + if [[ "${PROJECT}" == "hbase-thirdparty" ]]; then + jira_fix_version="thirdparty-${RELEASE_VERSION}" + elif [[ "${PROJECT}" =~ ^hbase- ]]; then jira_fix_version="${PROJECT}-${RELEASE_VERSION}" fi shopt -u nocasematch - update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" "${PREV_VERSION}" + update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" cd "${PROJECT}" @@ -302,15 +304,31 @@ if [[ "$1" == "publish-release" ]]; then mvn_log="${BASE_DIR}/mvn_deploy_release.log" log "Staging release in nexus" maven_deploy release "$mvn_log" - declare staged_repo_id="dryrun-no-repo" + declare staged_repo_id + declare hadoop3_staged_repo_id if ! is_dry_run; then - staged_repo_id=$(grep -o "Closing staging repository with ID .*" "$mvn_log" \ + mapfile -t staged_repo_ids < <(grep -o "Closing staging repository with ID .*" "$mvn_log" \ | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/') - log "Release artifacts successfully published to repo ${staged_repo_id}" + log "Release artifacts successfully published to repo: " "${staged_repo_ids[@]}" + repo_count="${#staged_repo_ids[@]}" + if [[ "${repo_count}" == "2" ]]; then + staged_repo_id=${staged_repo_ids[0]} + hadoop3_staged_repo_id=${staged_repo_ids[1]} + elif [[ "${repo_count}" == "1" ]]; then + staged_repo_id=${staged_repo_ids[0]} + hadoop3_staged_repo_id="not-applicable" + else + staged_repo_id="not-applicable" + hadoop3_staged_repo_id="not-applicable" + fi rm "$mvn_log" else log "Dry run: Release artifacts successfully built, but not published due to dry run." + staged_repo_id="dryrun-no-repo" + hadoop3_staged_repo_id="dryrun-no-repo" fi + export staged_repo_id + export hadoop3_staged_repo_id # Dump out email to send. Where we find vote.tmpl depends # on where this script is run from PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g' diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 00ff6cdcf51..787ec4dac18 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -75,7 +75,7 @@ function banner { } function log { - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ${1}" + echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") $*" } # current number of seconds since epoch @@ -298,12 +298,11 @@ EOF fi GPG_ARGS=("${GPG_ARGS[@]}" --local-user "${GPG_KEY}") - if ! is_dry_run; then - if [ -z "$ASF_PASSWORD" ]; then - stty -echo && printf "ASF_PASSWORD: " && read -r ASF_PASSWORD && printf '\n' && stty echo - fi - else - ASF_PASSWORD="***INVALID***" + # The nexus staging plugin needs the password to contact to remote server even if + # skipRemoteStaging is set to true, not sure why so here we need the password even + # if this is a dry run + if [ -z "$ASF_PASSWORD" ]; then + stty -echo && printf "ASF_PASSWORD: " && read -r ASF_PASSWORD && printf '\n' && stty echo fi export ASF_PASSWORD @@ -561,7 +560,6 @@ function get_jira_name { function update_releasenotes { local project_dir="$1" local jira_fix_version="$2" - local previous_jira_fix_version="$3" local jira_project local timing_token timing_token="$(start_step)" @@ -583,7 +581,7 @@ function update_releasenotes { "${project_dir}/CHANGES.md" || true else # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist - curl --location --fail --silent --show-error --output ${project_dir}/CHANGES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/CHANGES.md" + curl --location --fail --silent --show-error --output ${project_dir}/CHANGES.md "https://archive.apache.org/dist/hbase/${PREV_VERSION}/CHANGES.md" fi if [ -f "${project_dir}/RELEASENOTES.md" ]; then sed -i -e \ @@ -591,7 +589,7 @@ function update_releasenotes { "${project_dir}/RELEASENOTES.md" || true else # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist - curl --location --fail --silent --show-error --output ${project_dir}/RELEASENOTES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/RELEASENOTES.md" + curl --location --fail --silent --show-error --output ${project_dir}/RELEASENOTES.md "https://archive.apache.org/dist/hbase/${PREV_VERSION}/RELEASENOTES.md" fi # Yetus will not generate CHANGES if no JIRAs fixed against the release version @@ -645,6 +643,48 @@ make_src_release() { stop_step "${timing_token}" } +build_release_binary() { + local project="${1}" + local version="${2}" + local base_name="${project}-${version}" + local extra_flags=() + if [[ "${version}" = *-hadoop3 ]] || [[ "${version}" = *-hadoop3-SNAPSHOT ]]; then + extra_flags=("-Drevision=${version}" "-Dhadoop.profile=3.0") + fi + + cd "$project" || exit + git clean -d -f -x + # Three invocations of maven. This seems to work. One to + # populate the repo, another to build the site, and then + # a third to assemble the binary artifact. Trying to do + # all in the one invocation fails; a problem in our + # assembly spec to in maven. TODO. Meantime, three invocations. + cmd=("${MVN[@]}" "${extra_flags[@]}" clean install -DskipTests) + echo "${cmd[*]}" + "${cmd[@]}" + cmd=("${MVN[@]}" "${extra_flags[@]}" site -DskipTests) + echo "${cmd[*]}" + "${cmd[@]}" + kick_gpg_agent + cmd=("${MVN[@]}" "${extra_flags[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}") + echo "${cmd[*]}" + "${cmd[@]}" + + # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. + local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}" + if ls "${f_bin_prefix}"*-bin.tar.gz &>/dev/null; then + cp "${f_bin_prefix}"*-bin.tar.gz .. + cd .. || exit + for i in "${base_name}"*-bin.tar.gz; do + "${GPG}" "${GPG_ARGS[@]}" --armour --output "${i}.asc" --detach-sig "${i}" + "${GPG}" "${GPG_ARGS[@]}" --print-md SHA512 "${i}" > "${i}.sha512" + done + else + cd .. || exit + log "No ${f_bin_prefix}*-bin.tar.gz product; expected?" + fi +} + # Make binary release. # Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools # -- and then the version string. Expects to find checkout adjacent to this script @@ -662,37 +702,10 @@ make_binary_release() { local timing_token timing_token="$(start_step)" rm -rf "${base_name}"-bin* - cd "$project" || exit - git clean -d -f -x - # Three invocations of maven. This seems to work. One to - # populate the repo, another to build the site, and then - # a third to assemble the binary artifact. Trying to do - # all in the one invocation fails; a problem in our - # assembly spec to in maven. TODO. Meantime, three invocations. - cmd=("${MVN[@]}" clean install -DskipTests) - echo "${cmd[*]}" - "${cmd[@]}" - cmd=("${MVN[@]}" site -DskipTests) - echo "${cmd[*]}" - "${cmd[@]}" - kick_gpg_agent - cmd=("${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}") - echo "${cmd[*]}" - "${cmd[@]}" - - # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. - local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}" - if ls "${f_bin_prefix}"*-bin.tar.gz &>/dev/null; then - cp "${f_bin_prefix}"*-bin.tar.gz .. - cd .. || exit - for i in "${base_name}"*-bin.tar.gz; do - "${GPG}" "${GPG_ARGS[@]}" --armour --output "${i}.asc" --detach-sig "${i}" - "${GPG}" "${GPG_ARGS[@]}" --print-md SHA512 "${i}" > "${i}.sha512" - done - else - cd .. || exit - log "No ${f_bin_prefix}*-bin.tar.gz product; expected?" + build_release_binary "${project}" "${version}" + if should_build_with_hadoop3 "$project/pom.xml"; then + build_release_binary "${project}" "$(get_hadoop3_version "${version}")" fi stop_step "${timing_token}" @@ -746,6 +759,7 @@ function maven_deploy { #inputs: # Invoke with cwd=$PROJECT local deploy_type="$1" local mvn_log_file="$2" #secondary log file used later to extract staged_repo_id + local staging_dir if [[ "$deploy_type" != "snapshot" && "$deploy_type" != "release" ]]; then error "unrecognized deploy type, must be 'snapshot'|'release'" fi @@ -759,6 +773,8 @@ function maven_deploy { #inputs: elif [[ "$deploy_type" == "release" ]] && [[ "$RELEASE_VERSION" =~ SNAPSHOT ]]; then error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$RELEASE_VERSION'" fi + # Just output to parent directory, the staging directory has a staging prefix already + staging_dir="$(dirname "$(pwd)")/local-staged" # Publish ${PROJECT} to Maven repo # shellcheck disable=SC2154 log "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" @@ -767,20 +783,44 @@ function maven_deploy { #inputs: maven_set_version "$RELEASE_VERSION" # Prepare for signing kick_gpg_agent - declare -a mvn_goals=(clean) - if ! is_dry_run; then - mvn_goals=("${mvn_goals[@]}" deploy) + declare -a mvn_extra_flags=() + if is_dry_run; then + # In dry run mode, skip deploying to remote repo + mvn_extra_flags=("${mvn_extra_flags[@]}" -DskipRemoteStaging) fi - log "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" "${mvn_goals[@]}" + log "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true \ + -DaltStagingDirectory="${staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}" log "Logging to ${mvn_log_file}. This will take a while..." rm -f "$mvn_log_file" # The tortuous redirect in the next command allows mvn's stdout and stderr to go to mvn_log_file, # while also sending stderr back to the caller. # shellcheck disable=SC2094 - if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ - "${mvn_goals[@]}" 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then + if ! "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + -DaltStagingDirectory="${staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}" \ + 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then error "Deploy build failed, for details see log at '$mvn_log_file'." fi + local hadoop3_version + if should_build_with_hadoop3 pom.xml; then + hadoop3_version="$(get_hadoop3_version "${RELEASE_VERSION}")" + hadoop3_staging_dir="${staging_dir}-hadoop3" + log "Deploying artifacts for hadoop3..." + log "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true \ + -Drevision="${hadoop3_version}" -Dhadoop.profile=3.0 \ + -DaltStagingDirectory="${hadoop3_staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}" + { + echo "========================================================================" + echo "Deploy build for hadoop3" + echo "========================================================================" + } >> "$mvn_log_file" + # shellcheck disable=SC2094 + if ! "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + -Drevision="${hadoop3_version}" -Dhadoop.profile=3.0 \ + -DaltStagingDirectory="${hadoop3_staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}" \ + 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then + error "Deploy build failed, for details see log at '$mvn_log_file'." + fi + fi log "BUILD SUCCESS." stop_step "${timing_token}" return 0 @@ -798,3 +838,36 @@ function is_tracked() { git ls-files --error-unmatch "$file" &>/dev/null return $? } + +# When we have all the below conditions matched, we will build hadoop3 binaries +# 1. Use $revision place holder as version in pom +# 2. Has a hadoop-2.0 profile +# 3. Has a hadoop-3.0 profile +function should_build_with_hadoop3() { + local pom="$1" + maven_version="$(parse_version < "${pom}")" + # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html + # If we use ${revision} as placeholder, the way to bump maven version will be different + # shellcheck disable=SC2016 + if [[ "${maven_version}" != '${revision}' ]]; then + return 1 + fi + if ! xmllint --xpath "//*[local-name()='project']/*[local-name()='profiles']/*[local-name()='profile']/*[local-name()='id']/text()" "${pom}" \ + | grep -q ^hadoop-2.0$; then + return 1 + fi + if ! xmllint --xpath "//*[local-name()='project']/*[local-name()='profiles']/*[local-name()='profile']/*[local-name()='id']/text()" "${pom}" \ + | grep -q ^hadoop-3.0$; then + return 1 + fi + return 0 +} + +function get_hadoop3_version() { + local version="$1" + if [[ "${version}" =~ -SNAPSHOT$ ]]; then + echo "${version/-SNAPSHOT/-hadoop3-SNAPSHOT}" + else + echo "${version}-hadoop3" + fi +} diff --git a/dev-support/create-release/vote.tmpl b/dev-support/create-release/vote.tmpl index 49c201410b1..bdc7901f712 100644 --- a/dev-support/create-release/vote.tmpl +++ b/dev-support/create-release/vote.tmpl @@ -23,6 +23,10 @@ Maven artifacts are available in a staging repository at: https://repository.apache.org/content/repositories/${staged_repo_id}/ +Maven artifacts for hadoop3 are available in a staging repository at: + + https://repository.apache.org/content/repositories/${hadoop3_staged_repo_id}/ + Artifacts were signed with the ${GPG_KEY} key which can be found in: https://downloads.apache.org/hbase/KEYS