diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh new file mode 100755 index 00000000000..124bda561de --- /dev/null +++ b/dev-support/create-release/do-release-docker.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Creates a HBase release candidate. The script will update versions, tag the branch, +# build HBase binary packages and documentation, and upload maven artifacts to a staging +# repository. There is also a dry run mode where only local builds are performed, and +# nothing is uploaded to the ASF repos. +# +# Run with "-h" for options. For example, running below will do all +# steps above using the 'rm' dir under Downloads as workspace: +# +# $ ./do-release-docker.sh -d ~/Downloads/rm +# +# The scripts in this directory came originally from spark [1]. They were then +# modified to suite the hbase context. These scripts supercedes the old +# ../make_rc.sh script for making release candidates because what is here is more +# comprehensive doing more steps of the RM process as well as running in a +# container so the RM build environment can be a constant. +# +# It: +# * Tags release +# * Sets version to the release version +# * Sets version to next SNAPSHOT version. +# * Builds, signs, and hashes all artifacts. +# * Pushes release tgzs to the dev dir in a apache dist. +# * Pushes to repository.apache.org staging. +# +# The entry point is here, in the do-release-docker.sh script. +# +# 1. https://github.com/apache/spark/tree/master/dev/create-release +# +set -x +set -e +SELF=$(cd $(dirname $0) && pwd) +. "$SELF/release-util.sh" + +function usage { + local NAME=$(basename $0) + cat < "$GPG_KEY_FILE" + +run_silent "Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \ + docker build -t "hbase-rm:$IMGTAG" --build-arg UID=$UID "$SELF/hbase-rm" + +# Write the release information to a file with environment variables to be used when running the +# image. +ENVFILE="$WORKDIR/env.list" +fcreate_secure "$ENVFILE" + +function cleanup { + rm -f "$ENVFILE" + rm -f "$GPG_KEY_FILE" +} + +trap cleanup EXIT + +cat > $ENVFILE <> $ENVFILE + JAVA_VOL="--volume $JAVA:/opt/hbase-java" +fi + +echo "Building $RELEASE_TAG; output will be at $WORKDIR/output" +docker run -ti \ + --env-file "$ENVFILE" \ + --volume "$WORKDIR:/opt/hbase-rm" \ + $JAVA_VOL \ + "hbase-rm:$IMGTAG" diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh new file mode 100755 index 00000000000..30040ebfb7d --- /dev/null +++ b/dev-support/create-release/do-release.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Called by do-release-docker.sh. Can be run standalone but needs some love +# for it to work smooth. +set -x +SELF=$(cd $(dirname $0) && pwd) +. "$SELF/release-util.sh" + +while getopts "bn" opt; do + case $opt in + b) GIT_BRANCH=$OPTARG ;; + n) DRY_RUN=1 ;; + ?) error "Invalid option: $OPTARG" ;; + esac +done + +if [ "$RUNNING_IN_DOCKER" = "1" ]; then + # Run gpg agent. + eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400) + echo "GPG Version: `gpg --version`" + # Inside docker, need to import the GPG key stored in the current directory. + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key" + + # We may need to adjust the path since JAVA_HOME may be overridden by the driver script. + if [ -n "$JAVA_HOME" ]; then + export PATH="$JAVA_HOME/bin:$PATH" + else + # JAVA_HOME for the openjdk package. + export JAVA_HOME=/usr + fi +else + # Outside docker, need to ask for information about the release. + get_release_info +fi +export GPG_TTY=$(tty) + +function should_build { + local WHAT=$1 + [ -z "$RELEASE_STEP" ] || [ "$WHAT" = "$RELEASE_STEP" ] +} + +if should_build "tag" && [ $SKIP_TAG = 0 ]; then + run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ + "$SELF/release-tag.sh" + echo "It may take some time for the tag to be synchronized to github." + echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available." + read +else + echo "Skipping tag creation for $RELEASE_TAG." +fi + +if should_build "build"; then + run_silent "Building HBase..." "build.log" \ + "$SELF/release-build.sh" build +else + echo "Skipping build step." +fi + +if should_build "publish"; then + run_silent "Publishing release" "publish.log" \ + "$SELF/release-build.sh" publish-release +else + echo "Skipping publish step." +fi diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile new file mode 100644 index 00000000000..092b4183add --- /dev/null +++ b/dev-support/create-release/hbase-rm/Dockerfile @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Image for building HBase releases. Based on Ubuntu 16.04. +# +# Includes: +# * Java 8 +FROM ubuntu:18.04 + +# These arguments are just for reuse and not really meant to be customized. +ARG APT_INSTALL="apt-get install --no-install-recommends -y" + +# Install extra needed repos and refresh. +# +# This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch +# the most current package versions (instead of potentially using old versions cached by docker). +RUN apt-get clean && \ + apt-get update && \ + # Install openjdk 8. + $APT_INSTALL openjdk-8-jdk && \ + update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java && \ + # Install build / source control tools + $APT_INSTALL curl gnupg python-pip wget git maven subversion lsof \ + libcurl4-openssl-dev libxml2-dev && \ + pip install python-dateutil + +WORKDIR /opt/hbase-rm/output + +ARG UID +RUN useradd -m -s /bin/bash -p hbase-rm -u $UID hbase-rm +USER hbase-rm:hbase-rm + +ENTRYPOINT [ "/opt/hbase-rm/do-release.sh" ] diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh new file mode 100755 index 00000000000..a34189c8320 --- /dev/null +++ b/dev-support/create-release/release-build.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + +trap cleanup EXIT + +SELF=$(cd $(dirname $0) && pwd) +. "$SELF/release-util.sh" + +function exit_with_usage { + cat << EOF +usage: release-build.sh +Creates build deliverables from an HBase commit. + +Top level targets are + build: Create binary packages and commit them to dist.apache.org/repos/dist/dev/hbase/ + publish-snapshot: Publish snapshot release to Apache snapshots + publish-release: Publish a release to Apache release repo + +All other inputs are environment variables + +GIT_REF - Release tag or commit to build from +HBASE_PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2RC1) +HBASE_VERSION - (optional) Version of HBase being built (e.g. 2.1.2) + +ASF_USERNAME - Username of ASF committer account +ASF_PASSWORD - Password of ASF committer account + +GPG_KEY - GPG key used to sign release artifacts +GPG_PASSPHRASE - Passphrase for GPG key +EOF + exit 1 +} + +set -e + +function cleanup { + rm ${tmp_settings} &> /dev/null || true +} + +if [ $# -eq 0 ]; then + exit_with_usage +fi + +if [[ $@ == *"help"* ]]; then + exit_with_usage +fi + +if [[ -z "$ASF_PASSWORD" ]]; then + echo 'The environment variable ASF_PASSWORD is not set. Enter the password.' + echo + stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo +fi + +if [[ -z "$GPG_PASSPHRASE" ]]; then + echo 'The environment variable GPG_PASSPHRASE is not set. Enter the passphrase to' + echo 'unlock the GPG signing key that will be used to sign the release!' + echo + stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo + export GPG_PASSPHRASE + export GPG_TTY=$(tty) +fi + +for env in ASF_USERNAME GPG_PASSPHRASE GPG_KEY; do + if [ -z "${!env}" ]; then + echo "ERROR: $env must be set to run this script" + exit_with_usage + fi +done + +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 + +# Commit ref to checkout when building +GIT_REF=${GIT_REF:-master} + +RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase" + +GPG="gpg --pinentry-mode loopback -u $GPG_KEY --no-tty --batch" +NEXUS_ROOT=https://repository.apache.org/service/local/staging +NEXUS_PROFILE=8e226b97c0c82 # Profile for HBase staging uploads via INFRA-17900 Need nexus "staging profile id" for the hbase project +BASE_DIR=$(pwd) + +init_java +init_mvn +init_python +# Print out subset of perl version. +perl --version | grep 'This is' + +rm -rf hbase +git clone "$ASF_REPO" +cd hbase +git checkout $GIT_REF +git_hash=`git rev-parse --short HEAD` +echo "Checked out HBase git hash $git_hash" + +if [ -z "$HBASE_VERSION" ]; then + # Run $MVN in a separate command so that 'set -e' does the right thing. + TMP=$(mktemp) + $MVN help:evaluate -Dexpression=project.version > $TMP + HBASE_VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download) + rm $TMP +fi + +# Profiles for publishing snapshots and release to Maven Central +PUBLISH_PROFILES="-Papache-release -Prelease" + +if [[ ! $HBASE_VERSION < "2.0." ]]; then + if [[ $JAVA_VERSION < "1.8." ]]; then + echo "Java version $JAVA_VERSION is less than required 1.8 for 2.0+" + echo "Please set JAVA_HOME correctly." + exit 1 + fi +else + if ! [[ $JAVA_VERSION =~ 1\.7\..* ]]; then + if [ -z "$JAVA_7_HOME" ]; then + echo "Java version $JAVA_VERSION is higher than required 1.7 for pre-2.0" + echo "Please set JAVA_HOME correctly." + exit 1 + else + export JAVA_HOME="$JAVA_7_HOME" + fi + fi +fi + +# This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins +# machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion +# in its pull request. +LSOF=lsof +if ! hash $LSOF 2>/dev/null; then + LSOF=/usr/sbin/lsof +fi + +if [ -z "$HBASE_PACKAGE_VERSION" ]; then + HBASE_PACKAGE_VERSION="${HBASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" +fi + +DEST_DIR_NAME="$HBASE_PACKAGE_VERSION" + +git clean -d -f -x +cd .. + +tmp_repo=`pwd`/$(mktemp -d hbase-repo-XXXXX) +# Reexamine. Not sure this working. Pass as arg? That don't seem to work either! +tmp_settings="/${tmp_repo}/tmp-settings.xml" +echo "" > $tmp_settings +echo "apache.snapshots.https$ASF_USERNAME" >> $tmp_settings +echo "$ASF_PASSWORD" >> $tmp_settings +echo "apache-release$ASF_USERNAME" >> $tmp_settings +echo "$ASF_PASSWORD" >> $tmp_settings +echo "" >> $tmp_settings +echo "" >> $tmp_settings +export tmp_settings + +if [[ "$1" == "build" ]]; then + # Source and binary tarballs + echo "Packaging release source tarballs" + # Tar up the src and sign and hash it. + rm -rf hbase-$HBASE_VERSION-src* + ls + cd hbase + git clean -d -f -x + git archive --format=tar.gz --output=../hbase-$HBASE_VERSION-src.tar.gz --prefix=hbase-"${HBASE_VERSION}/" "${GIT_REF}" + cd .. + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour --output hbase-$HBASE_VERSION-src.tar.gz.asc \ + --detach-sig hbase-$HBASE_VERSION-src.tar.gz + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --print-md \ + SHA512 hbase-$HBASE_VERSION-src.tar.gz > hbase-$HBASE_VERSION-src.tar.gz.sha512 + + + # Add timestamps to mvn logs. + MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}" + + # Updated for each binary build; spark did many. HBase does one bin only. + make_binary_release() { + + echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Building binary dist" + cd hbase + + # Get maven home set by MVN + MVN_HOME=`$MVN -version 2>&1 | grep 'Maven home' | awk '{print $NF}'` + + echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Assembly" + git clean -d -f -x + # Three invocations of maven. This seems to work. One to + # populate the repo, another to build the site, and then + # a third to assemble the binary artifact. Trying to do + # all in the one invocation fails; a problem in our + # assembly spec to in maven. TODO. Meantime, three invocations. + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \ + clean install -DskipTests \ + -Dmaven.repo.local=${tmp_repo} + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \ + site -DskipTests \ + -Dmaven.repo.local=${tmp_repo} + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \ + install assembly:single -DskipTests \ + -Dcheckstyle.skip=true ${PUBLISH_PROFILES} \ + -Dmaven.repo.local=${tmp_repo} + + echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Copying and signing regular binary distribution" + cp ./hbase-assembly/target/hbase-$HBASE_VERSION*-bin.tar.gz .. + cd .. + for i in `ls hbase-$HBASE_VERSION*-bin.tar.gz`; do + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour \ + --output $i.asc --detach-sig $i + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --print-md \ + SHA512 $i > $i.sha512 + done + } + + make_binary_release + + if ! is_dry_run; then + svn co --depth=empty $RELEASE_STAGING_LOCATION svn-hbase + rm -rf "svn-hbase/${DEST_DIR_NAME}" + mkdir -p "svn-hbase/${DEST_DIR_NAME}" + + echo "Copying release tarballs" + cp hbase-*.tar.* "svn-hbase/${DEST_DIR_NAME}/" + cp hbase/CHANGES.md "svn-hbase/${DEST_DIR_NAME}/" + cp hbase/RELEASENOTES.md "svn-hbase/${DEST_DIR_NAME}/" + # This script usually reports an errcode along w/ the report. + generate_api_report ./hbase "${API_DIFF_TAG}" "${HBASE_PACKAGE_VERSION}" || true + cp api*.html "svn-hbase/${DEST_DIR_NAME}/" + + svn add "svn-hbase/${DEST_DIR_NAME}" + + cd svn-hbase + svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache HBase $HBASE_PACKAGE_VERSION" --no-auth-cache + cd .. + rm -rf svn-hbase + fi + + exit 0 +fi + +if [[ "$1" == "publish-snapshot" ]]; then + cd hbase + # Publish HBase to Maven release repo + echo "Deploying HBase SNAPSHOT at '$GIT_REF' ($git_hash)" + echo "Publish version is $HBASE_VERSION" + if [[ ! $HBASE_VERSION == *"SNAPSHOT"* ]]; then + echo "ERROR: Snapshots must have a version containing SNAPSHOT" + echo "ERROR: You gave version '$HBASE_VERSION'" + exit 1 + fi + # Coerce the requested version + $MVN versions:set -DnewVersion=$HBASE_VERSION + $MVN --settings $tmp_settings -DskipTests $PUBLISH_PROFILES deploy + cd .. + exit 0 +fi + +if [[ "$1" == "publish-release" ]]; then + cd hbase + # Publish HBase to Maven release repo + echo "Publishing HBase checkout at '$GIT_REF' ($git_hash)" + echo "Publish version is $HBASE_VERSION" + # Coerce the requested version + $MVN versions:set -DnewVersion=$HBASE_VERSION + MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \ + clean install -DskipTests \ + -Dcheckstyle.skip=true ${PUBLISH_PROFILES} \ + -Dmaven.repo.local=${tmp_repo} + pushd $tmp_repo/org/apache/hbase + # Remove any extra files generated during install + # Do find in hbase* because thirdparty is at same level! + find hbase* -type f | grep -v \.jar | grep -v \.pom | xargs rm + + # Using Nexus API documented here: + # https://support.sonatype.com/entries/39720203-Uploading-to-a-Staging-Repository-via-REST-API + if ! is_dry_run; then + echo "Creating Nexus staging repository" + repo_request="Apache HBase $HBASE_VERSION (commit $git_hash)" + out=$(curl -X POST -d "$repo_request" -u $ASF_USERNAME:$ASF_PASSWORD \ + -H "Content-Type:application/xml" -v \ + $NEXUS_ROOT/profiles/$NEXUS_PROFILE/start) + staged_repo_id=$(echo $out | sed -e "s/.*\(orgapachehbase-[0-9]\{4\}\).*/\1/") + echo "Created Nexus staging repository: $staged_repo_id" + fi + + # this must have .asc, and .sha1 - it really doesn't like anything else there + for file in $(find hbase* -type f) + do + if [[ "$file" == *.asc ]]; then + continue + fi + if [ ! -f $file.asc ]; then + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --output $file.asc \ + --detach-sig --armour $file; + fi + if [ $(command -v md5) ]; then + # Available on OS X; -q to keep only hash + md5 -q $file > $file.md5 + else + # Available on Linux; cut to keep only hash + md5sum $file | cut -f1 -d' ' > $file.md5 + fi + sha1sum $file | cut -f1 -d' ' > $file.sha1 + done + + if ! is_dry_run; then + nexus_upload=$NEXUS_ROOT/deployByRepositoryId/$staged_repo_id + echo "Uplading files to $nexus_upload" + for file in $(find hbase* -type f) + do + # strip leading ./ + file_short=$(echo $file | sed -e "s/\.\///") + dest_url="$nexus_upload/org/apache/hbase/$file_short" + echo " Uploading $file to $dest_url" + curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $file_short $dest_url + done + + echo "Closing nexus staging repository" + repo_request="$staged_repo_idApache HBase $HBASE_VERSION (commit $git_hash)" + out=$(curl -X POST -d "$repo_request" -u $ASF_USERNAME:$ASF_PASSWORD \ + -H "Content-Type:application/xml" -v \ + $NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish) + echo "Closed Nexus staging repository: $staged_repo_id" + fi + + popd + rm -rf $tmp_repo + cd .. + # Dump out email to send. + eval "echo \"$(< ../vote.tmpl)\"" |tee vote.txt + exit 0 +fi + +cd .. +rm -rf hbase +echo "ERROR: expects to be called with 'install', 'publish-release' or 'publish-snapshot'" diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh new file mode 100755 index 00000000000..166fb318e75 --- /dev/null +++ b/dev-support/create-release/release-tag.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Tags release. Updates releasenotes and changes. +SELF=$(cd $(dirname $0) && pwd) +. "$SELF/release-util.sh" + +function exit_with_usage { + local NAME=$(basename $0) + cat << EOF +usage: $NAME +Tags an HBase release on a particular branch. + +Inputs are specified with the following environment variables: +ASF_USERNAME - Apache Username +ASF_PASSWORD - Apache Password +GIT_NAME - Name to use with git +GIT_EMAIL - E-mail address to use with git +GIT_BRANCH - Git branch on which to make release +RELEASE_VERSION - Version used in pom files for release +RELEASE_TAG - Name of release tag +NEXT_VERSION - Development version after release +EOF + exit 1 +} + +set -e +set -o pipefail + +if [[ $@ == *"help"* ]]; then + exit_with_usage +fi + +if [[ -z "$ASF_PASSWORD" ]]; then + echo 'The environment variable ASF_PASSWORD is not set. Enter the password.' + echo + stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo +fi + +for env in ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH; do + if [ -z "${!env}" ]; then + echo "$env must be set to run this script" + exit 1 + fi +done + +init_java +init_mvn + +ASF_HBASE_REPO="gitbox.apache.org/repos/asf/hbase.git" + +rm -rf hbase +git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_HBASE_REPO" -b $GIT_BRANCH +update_releasenotes `pwd`/hbase "$RELEASE_VERSION" + +cd hbase + +git config user.name "$GIT_NAME" +git config user.email $GIT_EMAIL + +# Create release version +$MVN versions:set -DnewVersion=$RELEASE_VERSION | grep -v "no value" # silence logs +git add RELEASENOTES.md CHANGES.md + +git commit -a -m "Preparing HBase release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" +echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" +git tag $RELEASE_TAG + +# Create next version +$MVN versions:set -DnewVersion=$NEXT_VERSION | grep -v "no value" # silence logs + +git commit -a -m "Preparing development version $NEXT_VERSION" + +if ! is_dry_run; then + # Push changes + git push origin $RELEASE_TAG + git push origin HEAD:$GIT_BRANCH + cd .. + rm -rf hbase +else + cd .. + mv hbase hbase.tag + echo "Clone with version changes and tag available as hbase.tag in the output directory." +fi + diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh new file mode 100755 index 00000000000..f658906ddcc --- /dev/null +++ b/dev-support/create-release/release-util.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x +DRY_RUN=${DRY_RUN:-0} +GPG="gpg --pinentry-mode loopback --no-tty --batch" +ASF_REPO="https://gitbox.apache.org/repos/asf/hbase.git" + +ASF_REPO_WEBUI="https://gitbox.apache.org/repos/asf?p=hbase.git" +ASF_GITHUB_REPO="https://github.com/apache/hbase" +YETUS_VERSION=0.9.0 + +function error { + echo "$*" + exit 1 +} + +function read_config { + local PROMPT="$1" + local DEFAULT="$2" + local REPLY= + + read -p "$PROMPT [$DEFAULT]: " REPLY + local RETVAL="${REPLY:-$DEFAULT}" + if [ -z "$RETVAL" ]; then + error "$PROMPT is must be provided." + fi + echo "$RETVAL" +} + +function parse_version { + grep -e '.*' | \ + head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1 +} + +function run_silent { + local BANNER="$1" + local LOG_FILE="$2" + shift 2 + + echo "========================" + echo "= $BANNER" + echo "Command: $@" + echo "Log file: $LOG_FILE" + + "$@" 1>"$LOG_FILE" 2>&1 + + local EC=$? + if [ $EC != 0 ]; then + echo "Command FAILED. Check full logs for details." + tail "$LOG_FILE" + exit $EC + fi +} + +function fcreate_secure { + local FPATH="$1" + rm -f "$FPATH" + touch "$FPATH" + chmod 600 "$FPATH" +} + +function check_for_tag { + curl -s --head --fail "$ASF_GITHUB_REPO/releases/tag/$1" > /dev/null +} + +# API compare version. +function get_api_diff_version { + local version=$1 + local rev=$(echo "$version" | cut -d . -f 3) + local api_diff_tag + if [ $rev != 0 ]; then + local short_version=$(echo "$version" | cut -d . -f 1-2) + api_diff_tag="rel/${short_version}.$((rev - 1))" + else + local major=$(echo "$version" | cut -d . -f 1) + local minor=$(echo "$version" | cut -d . -f 2) + if [ $minor != 0 ]; then + api_diff_tag="rel/${major}.$((minor - 1)).0)" + else + api_diff_tag="rel/$((major - 1)).0.0)" + fi + fi + api_diff_tag=$(read_config "API diff tag", "$api_diff_tag") + echo $api_diff_tag +} + +# Get all branches that begin with 'branch-', the hbase convention for +# release branches, sort them and then pop off the most recent. +function get_release_info { + if [ -z "$GIT_BRANCH" ]; then + # If no branch is specified, find out the latest branch from the repo. + GIT_BRANCH=$(git ls-remote --heads "$ASF_REPO" | + grep refs/heads/branch- | + awk '{print $2}' | + sort -r | + head -n 1 | + cut -d/ -f3) + fi + + export GIT_BRANCH=$(read_config "Branch" "$GIT_BRANCH") + + # Find the current version for the branch. + local VERSION=$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | + parse_version) + echo "Current branch VERSION is $VERSION." + + if [[ ! $VERSION =~ .*-SNAPSHOT ]]; then + error "Not a SNAPSHOT version: $VERSION" + fi + + NEXT_VERSION="$VERSION" + RELEASE_VERSION="${VERSION/-SNAPSHOT/}" + SHORT_VERSION=$(echo "$VERSION" | cut -d . -f 1-2) + local REV=$(echo "$VERSION" | cut -d . -f 3) + + # Find out what RC is being prepared. + # - If the current version is "x.y.0", then this is RC0 of the "x.y.0" release. + # - If not, need to check whether the previous version has been already released or not. + # - If it has, then we're building RC0 of the current version. + # - If it has not, we're building the next RC of the previous version. + local RC_COUNT + if [ $REV != 0 ]; then + local PREV_REL_REV=$((REV - 1)) + PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}" + if check_for_tag "$PREV_REL_TAG"; then + RC_COUNT=0 + REV=$((REV + 1)) + NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT" + else + RELEASE_VERSION="${SHORT_VERSION}.${PREV_REL_REV}" + RC_COUNT=$(git ls-remote --tags "$ASF_REPO" "${RELEASE_VERSION}RC*" | wc -l) + # This makes a 'number' of it. + RC_COUNT=$((RC_COUNT)) + fi + else + REV=$((REV + 1)) + NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT" + RC_COUNT=0 + fi + + export NEXT_VERSION + export RELEASE_VERSION=$(read_config "Release" "$RELEASE_VERSION") + + + RC_COUNT=$(read_config "RC #" "$RC_COUNT") + + # Check if the RC already exists, and if re-creating the RC, skip tag creation. + RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}" + SKIP_TAG=0 + if check_for_tag "$RELEASE_TAG"; then + read -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER + if [ "$ANSWER" != "y" ]; then + error "Exiting." + fi + SKIP_TAG=1 + fi + + export RELEASE_TAG + + GIT_REF="$RELEASE_TAG" + if is_dry_run; then + echo "This is a dry run. Please confirm the ref that will be built for testing." + GIT_REF=$(read_config "Ref" "$GIT_REF") + fi + export GIT_REF + export HBASE_PACKAGE_VERSION="$RELEASE_TAG" + + export API_DIFF_TAG=$(get_api_diff_version $RELEASE_VERSION) + + # Gather some user information. + export ASF_USERNAME=$(read_config "ASF user" "$LOGNAME") + + GIT_NAME=$(git config user.name || echo "") + export GIT_NAME=$(read_config "Full name" "$GIT_NAME") + + export GIT_EMAIL="$ASF_USERNAME@apache.org" + export GPG_KEY=$(read_config "GPG key" "$GIT_EMAIL") + + cat <&1 | cut -d " " -f 2) + echo "java version: $JAVA_VERSION" + export JAVA_VERSION +} + +function init_python { + if ! [ -x "$(command -v python2)" ]; then + echo 'Error: python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' >&2 + exit 1 + fi + echo "python version: `python2 --version`" +} + +# Set MVN +function init_mvn { + if [ -n "$MAVEN_HOME" ]; then + MVN=${MAVEN_HOME}/bin/mvn + elif [ $(type -P mvn) ]; then + MVN=mvn + else + error "MAVEN_HOME is not set nor is mvn on the current path." + fi + echo "mvn version: `$MVN --version`" + # Add timestamped logging. + MVN="${MVN} -B" + export MVN +} + +# Writes report into cwd! +function generate_api_report { + local hbase=$1 + local previous_tag="$2" + local release_tag="$3" + # Generate api report. + ${hbase}/dev-support/checkcompatibility.py --annotation \ + org.apache.yetus.audience.InterfaceAudience.Public \ + $previous_tag $release_tag + local previous_version=$(echo ${previous_tag} | sed -e 's/rel\///') + cp ${hbase}/target/compat-check/report.html "./api_compare_${previous_version}_to_${release_tag}.html" +} + +# Update the CHANGES.md +# DOES NOT DO COMMITS! Caller should do that. +# yetus requires python2 to be on the path. +function update_releasenotes { + local hbase="$1" + local release_version="$2" + local yetus="apache-yetus-${YETUS_VERSION}" + wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/${yetus}-bin.tar.gz" | \ + tar xvz -C . + cd ./${yetus} + ./bin/releasedocmaker -p HBASE --fileversions -v ${release_version} -l --sortorder=newer --skip-credits + # First clear out the changes written by previous RCs. + sed -i -e "/^## Release ${release_version}/,/^## Release/ {//!d; /^## Release ${release_version}/d;}" \ + ${hbase}/CHANGES.md + sed -i -e "/^# HBASE ${release_version} Release Notes/,/^# HBASE/{//!d; /^# HBASE ${release_version} Release Notes/d;}" \ + ${hbase}/RELEASENOTES.md + + # The above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md. + # To insert into hbase CHANGES.me...need to cut the top off the + # CHANGELOG.X.X.X.md file removing license and first line and then + # insert it after the license comment closing where we have a + # DO NOT REMOVE marker text! + sed -i -e '/## Release/,$!d' CHANGELOG.${release_version}.md + sed -i -e "/DO NOT REMOVE/r CHANGELOG.${release_version}.md" ${hbase}/CHANGES.md + # Similar for RELEASENOTES but slightly different. + sed -i -e '/Release Notes/,$!d' RELEASENOTES.${release_version}.md + sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${release_version}.md" ${hbase}/RELEASENOTES.md + cd .. +} diff --git a/dev-support/create-release/vote.tmpl b/dev-support/create-release/vote.tmpl new file mode 100644 index 00000000000..d2f5398beb0 --- /dev/null +++ b/dev-support/create-release/vote.tmpl @@ -0,0 +1,28 @@ +Please vote on this Apache HBase release candidate (RC), ${RELEASE_VERSION}. + +The VOTE will remain open for at least 72 hours. + +[ ] +1 Release this package as Apache HBase ${RELEASE_VERSION} +[ ] -1 Do not release this package because ... + +The tag to be voted on is ${RELEASE_TAG}: + + https://github.com/apache/hbase/tree/${RELEASE_TAG} + +The release files, including signatures, digests, as well as CHANGES.md +and RELEASENOTES.md included in this RC can be found at: + + https://dist.apache.org/repos/dist/dev/hbase/${RELEASE_TAG}/ + +Maven artifacts are available in a staging repository at: + + https://repository.apache.org/content/repositories/${staged_repo_id}/ + +Artifacts were signed with the ${GPG_KEY} key which can be found in: + + https://dist.apache.org/repos/dist/release/hbase/KEYS + +To learn more about Apache HBase, please see http://hbase.apache.org/ + +Thanks, +Your HBase Release Manager diff --git a/pom.xml b/pom.xml index 7df8071acb0..38b777525c0 100755 --- a/pom.xml +++ b/pom.xml @@ -863,6 +863,7 @@ **/shaded/com/google/protobuf/** **/src/main/patches/** + **/vote.tmpl diff --git a/src/main/asciidoc/_chapters/developer.adoc b/src/main/asciidoc/_chapters/developer.adoc index 14a549cc86d..cc2181889e7 100644 --- a/src/main/asciidoc/_chapters/developer.adoc +++ b/src/main/asciidoc/_chapters/developer.adoc @@ -546,24 +546,179 @@ For the build to sign them for you, you a properly configured _settings.xml_ in [[maven.release]] === Making a Release Candidate -Only committers may make releases of hbase artifacts. +Only committers can make releases of hbase artifacts. + +Making a Release Candidate (RC) is comprised of many involved steps. +The steps are described in detail below in the section +<> but a script that automates the process is +described first in <>. .Before You Begin -Make sure your environment is properly set up. Maven and Git are the main tooling -used in the below. You'll need a properly configured _settings.xml_ file in your -local _~/.m2_ maven repository with logins for apache repos (See <>). -You will also need to have a published signing key. Browse the Hadoop -link:http://wiki.apache.org/hadoop/HowToRelease[How To Release] wiki page on -how to release. It is a model for most of the instructions below. It often has more -detail on particular steps, for example, on adding your code signing key to the -project KEYS file up in Apache or on how to update JIRA in preparation for release. - -Before you make a release candidate, do a practice run by deploying a SNAPSHOT. Check to be sure recent builds have been passing for the branch from where you are going to take your release. You should also have tried recent branch tips out on a cluster under load, perhaps by running the `hbase-it` integration test suite for a few hours to 'burn in' the near-candidate bits. +Make sure your environment is properly set up. + +You will need to have a published signing key added to the hbase +link:https://dist.apache.org/repos/dist/release/hbase/KEYS[KEYS] file. +Browse the Hadoop link:http://wiki.apache.org/hadoop/HowToRelease[How To Release] +wiki page on how to release. It is a model for the instructions below. It often has more +detail on particular steps. For example, _Step 1._ describes how to add +your code signing key to the +link:https://dist.apache.org/repos/dist/release/hbase/KEYS[KEYS] file up in Apache +(as well as outlinks on what a signing key is). + +Next make sure JIRA is properly primed, that all issues targeted against +the prospective release have been resolved and are present in git on the +particular branch. If any outstanding, move them out of the release by +adjusting the fix version to remove this pending release as a target. +Any JIRA with a fix version that matches the release candidate +target release will be included in the generated _CHANGES.md/RELEASENOTES.md_ +that ship with the release so make sure JIRA is correct before you begin. +(_Step 2._ in link:http://wiki.apache.org/hadoop/HowToRelease[How To Release] +talks of how it is done in Hadoop). + +[[do-release-docker.sh]] +==== Release Candidate Generating Script +The _dev-support/create-release/do-release-docker.sh_ Release Candidate (RC) +Generating script is maintained in the master branch but can generate RCs +for any 2.x+ branch (The script does not work against branch-1). Check out +and update the master branch when making RCs. + +The script builds in a Docker container to ensure we have a consistent +environment building. It will ask you for passwords for apache and for your +gpg signing key so it can sign and commit on your behalf. The passwords +are passed to gpg-agent in the container and purged along with the container +when the build is done. + +[NOTE] +==== +_dev-support/create-release/do-release-docker.sh_ supercedes the previous +_dev-support/make_rc.sh_ script. It is more comprehensive automating all +steps, rather than a portion, building a RC. +==== + +The script will: + + * Set version to the release version + * Updates RELEASENOTES.md and CHANGES.md + * Tag the RC + * Set version to next SNAPSHOT version. + * Builds, signs, and hashes all artifacts. + * Generates the api compatibility report + * Pushes release tgzs to the dev dir in a apache dist. + * Pushes to repository.apache.org staging. + * Creates vote email template. + +Pass _-h_ to _dev-support/create-release/do-release-docker.sh_ to +see available options: + +``` +Usage: do-release-docker.sh [options] + +This script runs the release scripts inside a docker image. + +Options: + + -d [path] required. working directory. output will be written to "output" in here. + -n dry run mode. Checks and local builds, but does not upload anything. + -t [tag] tag for the hbase-rm docker image to use for building (default: "latest"). + -j [path] path to local JDK installation to use building. By default the script will + use openjdk8 installed in the docker image. + -s [step] runs a single step of the process; valid steps are: tag, build, publish. if + none specified, runs tag, then build, and then publish. +``` + +Running the below command will do all steps above using the +'rm' working directory under Downloads as workspace: +``` + $ ./dev-support/create-release/do-release-docker.sh -d ~/Downloads/rm +``` + +The script will ask you a set of questions about the release version +and branch, the version to generate the compatibility report against, +and so on, before it starts executing (If you set the appropriate +environment variables, the script will skip asking you questions -- +which can come in handy if you end up having to re-run the script +multiple times). + +On branch 2.1, a Release Candidate (RC) creation can take many hours +(~8 hours) so run your build on a machine you know will be +around for this swath of time. Start the build inside a _screen_ +or _tmux_ session in case you become disconnected from your +build box. + +The build is made of three stages: tag, build, and +publish. If the script fails, you may resort to 'fixing' the +failure manually and then asking the script to run the +subsequent stage rather than start over. + +When the scripts run, they use the passed working directory. +Under the working directory is an _output_ dir. In here is +where the checkouts go, where we build up the _svn_ directory +to _svn_ commit to _apache/dist/dev_, etc. Each step also +dumps a log file in here: e.g. _tag.log_ for the tagging +step and _build.log_ for building. + +The _tagging_ step will checkout hbase, set the version number +in all the poms – e.g. if branch-2.0 is at 2.0.6-SNAPSHOT +and you are making a 2.0.5 RC, it will set the versions in +all poms to 2.0.5 – appropriately. It then generate CHANGES.md +and RELEASENOTES.md by checking out yetus and then +calling its generator scripts. It then commits the poms with +their new versions along with the changed CHANGES.md and +RELEASENOTES.md, tags, and pushes up all changes to the +apache repo. + +The _build_ step will checkout hbase, build all including +javadoc and doc (javadoc takes the bulk of the time – 4 hours plus), +run assemblies to produce src and bin tarballs, sign and hash it +all, and then make a dir under apache dist dev named for the RC. +It will copy all artifacts in here including top-level CHANGES.md +and RELEASENOTES.md. It will generate api diff docs and put them +into this RC dir too. When done, it commits the svn RC. + +The publish step will checkout hbase, build, and then copy up all +artifacts to repository.apache.org (signed and hashed). When done, +it will dump out an email template with all the correct links in place. + +Check the artifacts pushed to the dev distribution directory and up +in repository.apache.org. If all looks good, check the generated +email and send to the dev list. + +Under the create-release dir, scripts should make some sense: +``` +do-release-docker.sh # Main entrance. +do-release.sh . # More checks. Not usable really other than by setting env variables before running it. +release-tag.sh # Does tagging steps. +release-build.sh . # Does the build and publish step. +release-util.sh # Utility used by all of the above. +vote.tmpl # Template for email to send out. +hbase-rm # Has docker image we use. +``` + +If the RC fails, the script will do the right thing when it comes +to edit of the _CHANGES.md_ and _RELEASENOTES.md_ removing the old +and updating the files with the updated content (No harm verifying +though). + +One trick for checking stuff especially in utility is to do as follows: + +``` +$ source release-util.sh ; generate_api_report ../../ rel/2.1.3 2.14RC1 +``` + +i.e. source the release-util.sh script and then run one of its functions +passing args. Helped debugging stuff. + +[[rc_procedure]] +==== Release Candidate Procedure +Here we describe the steps involved generating a Release Candidate, the steps +automated by the script described in the previous section. + +The process below makes use of various tools, mainly _git_ and _maven_. .Specifying the Heap Space for Maven [NOTE] @@ -579,75 +734,60 @@ MAVEN_OPTS="-Xmx4g -XX:MaxPermSize=256m" mvn package You could also set this in an environment variable or alias in your shell. ==== +===== Update the _CHANGES.md_ and _RELEASENOTES.md_ files and the POM files. -[NOTE] -==== -The script _dev-support/make_rc.sh_ automates many of the below steps. -It will checkout a tag, clean the checkout, build src and bin tarballs, -and deploy the built jars to repository.apache.org. -It does NOT do the modification of the _CHANGES.txt_ for the release, -the checking of the produced artifacts to ensure they are 'good' -- -e.g. extracting the produced tarballs, verifying that they -look right, then starting HBase and checking that everything is running -correctly -- or the signing and pushing of the tarballs to -link:https://people.apache.org[people.apache.org]. -Take a look. Modify/improve as you see fit. -==== +Update _CHANGES.md_ with the changes since the last release. Be careful with where you put +headings and license. Respect the instructions and warning you find in current +_CHANGES.md_ and _RELEASENOTES.md_ since these two files are processed by tooling that is +looking for particular string sequences. See link:https://issues.apache.org/jira/browse/HBASE-21399[HBASE-21399] +for description on how to make use of yetus generating additions to +_CHANGES.md_ and _RELEASENOTES.md_ (RECOMMENDED!). Adding JIRA fixes, make sure the +URL to the JIRA points to the proper location which lists fixes for this release. -.Procedure: Release Procedure -. Update the _CHANGES.txt_ file and the POM files. -+ -Update _CHANGES.txt_ with the changes since the last release. -Make sure the URL to the JIRA points to the proper location which lists fixes for this release. -Adjust the version in all the POM files appropriately. +Next, adjust the version in all the POM files appropriately. If you are making a release candidate, you must remove the `-SNAPSHOT` label from all versions in all pom.xml files. If you are running this receipe to publish a snapshot, you must keep the `-SNAPSHOT` suffix on the hbase version. The link:http://www.mojohaus.org/versions-maven-plugin/[Versions Maven Plugin] can be of use here. To set a version in all the many poms of the hbase multi-module project, use a command like the following: -+ + [source,bourne] ---- $ mvn clean org.codehaus.mojo:versions-maven-plugin:2.5:set -DnewVersion=2.1.0-SNAPSHOT ---- -+ -Make sure all versions in poms are changed! Checkin the _CHANGES.txt_ and any maven version changes. -. Update the documentation. -+ +Make sure all versions in poms are changed! Checkin the _CHANGES.md_, _RELEASENOTES.md_, and +any maven version changes. + +===== Update the documentation. + Update the documentation under _src/main/asciidoc_. This usually involves copying the latest from master branch and making version-particular -adjustments to suit this release candidate version. +adjustments to suit this release candidate version. Commit your changes. -. Clean the checkout dir -+ +===== Clean the checkout dir [source,bourne] ---- - $ mvn clean $ git clean -f -x -d ---- - -. Run Apache-Rat +===== Run Apache-Rat Check licenses are good -+ + [source,bourne] ---- - -$ mvn apache-rat +$ mvn apache-rat:check ---- -+ + If the above fails, check the rat log. -+ [source,bourne] ---- $ grep 'Rat check' patchprocess/mvn_apache_rat.log ---- -+ -. Create a release tag. +===== Create a release tag. Presuming you have run basic tests, the rat check, passes and all is looking good, now is the time to tag the release candidate (You always remove the tag if you need to redo). To tag, do @@ -656,10 +796,8 @@ All tags should be signed tags; i.e. pass the _-s_ option (See link:http://https://git-scm.com/book/id/v2/Git-Tools-Signing-Your-Work[Signing Your Work] for how to set up your git environment for signing). -+ [source,bourne] ---- - $ git tag -s 2.0.0-alpha4-RC0 -m "Tagging the 2.0.0-alpha4 first Releae Candidate (Candidates start at zero)" ---- @@ -672,25 +810,22 @@ they are preserved in the Apache repo as in: ---- Push the (specific) tag (only) so others have access. -+ + [source,bourne] ---- - $ git push origin 2.0.0-alpha4-RC0 ---- -+ + For how to delete tags, see link:http://www.manikrathee.com/how-to-delete-a-tag-in-git.html[How to Delete a Tag]. Covers deleting tags that have not yet been pushed to the remote Apache repo as well as delete of tags pushed to Apache. - -. Build the source tarball. -+ +===== Build the source tarball. Now, build the source tarball. Lets presume we are building the source tarball for the tag _2.0.0-alpha4-RC0_ into _/tmp/hbase-2.0.0-alpha4-RC0/_ (This step requires that the mvn and git clean steps described above have just been done). -+ + [source,bourne] ---- $ git archive --format=tar.gz --output="/tmp/hbase-2.0.0-alpha4-RC0/hbase-2.0.0-alpha4-src.tar.gz" --prefix="hbase-2.0.0-alpha4/" $git_tag @@ -701,7 +836,7 @@ _/tmp/hbase-2.0.0-alpha4-RC0_ build output directory (We don't want the _RC0_ in These bits are currently a release candidate but if the VOTE passes, they will become the release so we do not taint the artifact names with _RCX_). -. Build the binary tarball. +===== Build the binary tarball. Next, build the binary tarball. Add the `-Prelease` profile when building. It runs the license apache-rat check among other rules that help ensure all is wholesome. Do it in two steps. @@ -710,7 +845,6 @@ First install into the local repository [source,bourne] ---- - $ mvn clean install -DskipTests -Prelease ---- @@ -720,24 +854,21 @@ documentation. [source,bourne] ---- - $ mvn install -DskipTests site assembly:single -Prelease ---- -+ Otherwise, the build complains that hbase modules are not in the maven repository when you try to do it all in one step, especially on a fresh repository. It seems that you need the install goal in both steps. -+ + Extract the generated tarball -- you'll find it under _hbase-assembly/target_ and check it out. Look at the documentation, see if it runs, etc. If good, copy the tarball beside the source tarball in the build output directory. +===== Deploy to the Maven Repository. -. Deploy to the Maven Repository. -+ Next, deploy HBase to the Apache Maven repository. Add the apache-release` profile when running the `mvn deploy` command. This profile comes from the Apache parent pom referenced by our pom files. @@ -746,43 +877,42 @@ _settings.xml_ is configured correctly, as described in <>. This step depends on the local repository having been populate by the just-previous bin tarball build. -+ + [source,bourne] ---- - $ mvn deploy -DskipTests -Papache-release -Prelease ---- -+ + This command copies all artifacts up to a temporary staging Apache mvn repository in an 'open' state. More work needs to be done on these maven artifacts to make them generally available. -+ + We do not release HBase tarball to the Apache Maven repository. To avoid deploying the tarball, do not include the `assembly:single` goal in your `mvn deploy` command. Check the deployed artifacts as described in the next section. .make_rc.sh [NOTE] ==== -If you run the _dev-support/make_rc.sh_ script, this is as far as it takes you. +If you ran the old _dev-support/make_rc.sh_ script, this is as far as it takes you. To finish the release, take up the script from here on out. ==== -. Make the Release Candidate available. -+ +===== Make the Release Candidate available. + The artifacts are in the maven repository in the staging area in the 'open' state. While in this 'open' state you can check out what you've published to make sure all is good. To do this, log in to Apache's Nexus at link:https://repository.apache.org[repository.apache.org] using your Apache ID. Find your artifacts in the staging repository. Click on 'Staging Repositories' and look for a new one ending in "hbase" with a status of 'Open', select it. Use the tree view to expand the list of repository contents and inspect if the artifacts you expect are present. Check the POMs. As long as the staging repo is open you can re-upload if something is missing or built incorrectly. -+ + If something is seriously wrong and you would like to back out the upload, you can use the 'Drop' button to drop and delete the staging repository. Sometimes the upload fails in the middle. This is another reason you might have to 'Drop' the upload from the staging repository. -+ + If it checks out, close the repo using the 'Close' button. The repository must be closed before a public URL to it becomes available. It may take a few minutes for the repository to close. Once complete you'll see a public URL to the repository in the Nexus UI. You may also receive an email with the URL. Provide the URL to the temporary staging repository in the email that announces the release candidate. (Folks will need to add this repo URL to their local poms or to their local _settings.xml_ file to pull the published release candidate artifacts.) -+ + When the release vote concludes successfully, return here and click the 'Release' button to release the artifacts to central. The release process will automatically drop and delete the staging repository. -+ + .hbase-downstreamer [NOTE] ==== @@ -793,11 +923,11 @@ Make sure you are pulling from the repository when tests run and that you are no ==== See link:https://www.apache.org/dev/publishing-maven-artifacts.html[Publishing Maven Artifacts] for some pointers on this maven staging process. -+ + If the HBase version ends in `-SNAPSHOT`, the artifacts go elsewhere. They are put into the Apache snapshots repository directly and are immediately available. Making a SNAPSHOT release, this is what you want to happen. -+ + At this stage, you have two tarballs in your 'build output directory' and a set of artifacts in a staging area of the maven repository, in the 'closed' state. Next sign, fingerprint and then 'stage' your release candiate build output directory via svnpubsub by committing @@ -808,7 +938,6 @@ link:https://dist.apache.org/repos/dist/release/hbase[release/hbase]). In the _v [source,bourne] ---- - $ for i in *.tar.gz; do echo $i; gpg --print-md MD5 $i > $i.md5 ; done $ for i in *.tar.gz; do echo $i; gpg --print-md SHA512 $i > $i.sha ; done $ for i in *.tar.gz; do echo $i; gpg --armor --output $i.asc --detach-sig $i ; done @@ -832,12 +961,11 @@ $ mv 0.96.0RC0 /Users/stack/checkouts/hbase.dist.dev.svn $ svn add 0.96.0RC0 $ svn commit ... ---- -+ + Ensure it actually gets published by checking link:https://dist.apache.org/repos/dist/dev/hbase/[https://dist.apache.org/repos/dist/dev/hbase/]. Announce the release candidate on the mailing list and call a vote. - [[maven.snapshot]] === Publishing a SNAPSHOT to maven