hbase/dev-support/create-release/release-build.sh

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

trap cleanup EXIT

# Source in utils.
SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=SCRIPTDIR/release-util.sh
. "$SELF/release-util.sh"

# Print usage and exit.
function exit_with_usage {
  cat <<'EOF'
Usage: release-build.sh <tag|publish-dist|publish-snapshot|publish-release>
Creates release deliverables from a tag or commit.
Argument: one of 'tag', 'publish-dist', 'publish-snapshot', or 'publish-release'
  tag               Prepares for release on specified git branch: Set release version,
                    update CHANGES and RELEASENOTES, create release tag,
                    increment version for ongoing dev, and publish to Apache git repo.
  publish-dist      Build and publish distribution packages (tarballs) to Apache dist repo
  publish-snapshot  Build and publish maven artifacts snapshot release to Apache snapshots repo
  publish-release   Build and publish maven artifacts release to Apache release repo, and
                    construct vote email from template

All other inputs are environment variables.  Please use do-release-docker.sh or
do-release.sh to set up the needed environment variables.  This script, release-build.sh,
is not intended to be called stand-alone, and such use is untested.  The env variables used are:

Used for 'tag' and 'publish' stages:
  PROJECT - The project to build. No default.
  RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2)
    Required for 'tag'; defaults for 'publish' to the version in pom at GIT_REF
  RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0), also used by
    publish-dist as package version name in dist directory path
  ASF_USERNAME - Username of ASF committer account
  ASF_PASSWORD - Password of ASF committer account
  DRY_RUN - 1:true (default), 0:false. If "1", does almost all the work, but doesn't actually
    publish anything to upstream source or object repositories. It defaults to "1", so if you want
    to actually publish you have to set '-f' (force) flag in do-release.sh or do-release-docker.sh.

Used only for 'tag':
  YETUS_HOME - installation location for Apache Yetus
  GIT_NAME - Name to use with git
  GIT_EMAIL - E-mail address to use with git
  GIT_BRANCH - Git branch on which to make release. Tag is always placed at HEAD of this branch.
  NEXT_VERSION - Development version after release (e.g. 2.1.3-SNAPSHOT)

Used only for 'publish':
  GIT_REF - Release tag or commit to build from (defaults to $RELEASE_TAG; only need to
    separately define GIT_REF if RELEASE_TAG is not actually present as a tag at publish time)
    If both RELEASE_TAG and GIT_REF are undefined it will default to HEAD of master.
  GPG_KEY - GPG key id (usually email addr) used to sign release artifacts
  REPO - Set to full path of a directory to use as maven local repo (dependencies cache)
    to avoid re-downloading dependencies for each stage.  It is automatically set if you
    request full sequence of stages (tag, publish-dist, publish-release) in do-release.sh.

For example:
 $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist
EOF
  exit 1
}

set -e

function cleanup {
  # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out.
  if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then
    log "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. Set REPO to reuse downloads." >&2
    rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true
    rm -rf "${MAVEN_LOCAL_REPO}" &> /dev/null || true
  fi
}

if [ $# -ne 1 ]; then
  exit_with_usage
fi

if [[ "$1" == "-h" ]]; then
  exit_with_usage
fi

if [[ "$*" == *"help"* ]]; then
  exit_with_usage
fi

init_locale
init_java
init_mvn
init_python
# Print out subset of perl version (used in git hooks and japi-compliance-checker)
perl --version | grep 'This is'

rm -rf "${PROJECT}"

if is_debug; then
  set -x  # detailed logging during action
fi

if [[ "$1" == "tag" ]]; then
  init_yetus
  # for 'tag' stage
  set -o pipefail
  check_get_passwords ASF_PASSWORD
  check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
  if [ -z "${GIT_REPO}" ]; then
    check_needed_vars ASF_USERNAME ASF_PASSWORD
  fi
  git_clone_overwrite

  # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
  # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
  # projects share the same HBASE jira name.  To make this work, by convention, the HBASE jira "Fix Version"
  # field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0".
  # So, here we prepend the project name to the version, but only for the hbase sub-projects.
  jira_fix_version="${RELEASE_VERSION}"
  shopt -s nocasematch
  if [[ "${PROJECT}" == "hbase-thirdparty" ]]; then
    jira_fix_version="thirdparty-${RELEASE_VERSION}"
  elif [[ "${PROJECT}" =~ ^hbase- ]]; then
    jira_fix_version="${PROJECT}-${RELEASE_VERSION}"
  fi
  shopt -u nocasematch
  update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}"

  cd "${PROJECT}"

  git config user.name "$GIT_NAME"
  git config user.email "$GIT_EMAIL"
  git config user.signingkey "${GPG_KEY}"

  # Create release version
  maven_set_version "$RELEASE_VERSION"
  find . -name pom.xml -exec git add {} \;
  # Always put CHANGES.md and RELEASENOTES.md to parent directory, so later we do not need to
  # check their position when generating release data. We can not put them under the source code
  # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
  # generating src release tarball, we will reset the git repo
  if is_tracked "CHANGES.md"; then
    git add RELEASENOTES.md CHANGES.md
    git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
    cp CHANGES.md ../
    cp RELEASENOTES.md ../
  else
    # CHANGES.md is not tracked, should 3.x+
    git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG"
    mv CHANGES.md ../
    mv RELEASENOTES.md ../
  fi

  log "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH"
  git tag -s -m "Via create-release" "$RELEASE_TAG"

  # Create next version
  maven_set_version "$NEXT_VERSION"
  find . -name pom.xml -exec git add {} \;
  git commit -s -m "Preparing development version $NEXT_VERSION"

  if ! is_dry_run; then
    # Push changes
    git push origin "$RELEASE_TAG"
    git push origin "HEAD:$GIT_BRANCH"
    cd ..
    rm -rf "${PROJECT}"
  else
    cd ..
    mv "${PROJECT}" "${PROJECT}.tag"
    log "Dry run: Clone with version changes and tag available as ${PROJECT}.tag in the output directory."
  fi
  exit 0
fi

### Below is for 'publish-*' stages ###
check_get_passwords ASF_PASSWORD
check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY

# Commit ref to checkout when building
BASE_DIR=$(pwd)
GIT_REF=${GIT_REF:-master}
if [[ "$PROJECT" =~ ^hbase ]]; then
  RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase"
else
  RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/${PROJECT}"
fi

# in case of dry run, enable publish steps to chain from tag step
if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
  ln -s "${PROJECT}.tag" "${PROJECT}"
else
  git_clone_overwrite
fi
cd "${PROJECT}"
git checkout "$GIT_REF"
git_hash="$(git rev-parse --short HEAD)"
GIT_LONG_HASH="$(git rev-parse HEAD)"
log "Checked out ${PROJECT} at ${GIT_REF} commit $git_hash"

if [ -z "${RELEASE_VERSION}" ]; then
  RELEASE_VERSION="$(maven_get_version)"
fi

# This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins
# machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion
# in its pull request.
LSOF=lsof
if ! hash $LSOF 2>/dev/null; then
  LSOF=/usr/sbin/lsof
fi

package_version_name="$RELEASE_TAG"
if [ -z "$package_version_name" ]; then
  package_version_name="${RELEASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}"
fi

git clean -d -f -x
cd ..
if [[ "$PROJECT" =~ ^hbase- ]]; then
  DEST_DIR_NAME="${PROJECT}-${package_version_name}"
else
  DEST_DIR_NAME="$package_version_name"
fi

if [[ "$1" == "publish-dist" ]]; then
  # Source and binary tarballs
  log "Packaging release source tarballs"
  make_src_release "${PROJECT}" "${RELEASE_VERSION}"

  # we do not have binary tarballs for hbase-thirdparty
  if [[ "${PROJECT}" != "hbase-thirdparty" ]]; then
    make_binary_release "${PROJECT}" "${RELEASE_VERSION}"
  fi

  svn_target="svn-${PROJECT}"
  svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target"
  rm -rf "${svn_target:?}/${DEST_DIR_NAME}"
  mkdir -p "$svn_target/${DEST_DIR_NAME}"

  log "Copying release tarballs"
  cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/"
  cp "CHANGES.md" "$svn_target/${DEST_DIR_NAME}/"
  cp "RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/"

  shopt -s nocasematch
  # Generate api report only if project is hbase for now.
  if [ "${PROJECT}" == "hbase" ]; then
    # This script usually reports an errcode along w/ the report.
    generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${GIT_REF}" || true
    cp api*.html "$svn_target/${DEST_DIR_NAME}/"
  fi
  shopt -u nocasematch

  log "svn add"
  svn add "$svn_target/${DEST_DIR_NAME}"

  if ! is_dry_run; then
    cd "$svn_target"
    svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $package_version_name" --no-auth-cache
    cd ..
    rm -rf "$svn_target"
  else
    mv "$svn_target/${DEST_DIR_NAME}" "${svn_target}_${DEST_DIR_NAME}.dist"
    log "Dry run: svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist"
    rm -rf "$svn_target"
  fi
  log "svn ci done"

  exit 0
fi

if [[ "$1" == "publish-snapshot" ]]; then
  (
  cd "${PROJECT}"
  mvn_log="${BASE_DIR}/mvn_deploy_snapshot.log"
  log "Publishing snapshot to nexus"
  maven_deploy snapshot "$mvn_log"
  if ! is_dry_run; then
    log "Snapshot artifacts successfully published to repo."
    rm "$mvn_log"
  else
    log "Dry run: Snapshot artifacts successfully built, but not published due to dry run."
  fi
  )
  exit $?
fi

if [[ "$1" == "publish-release" ]]; then
  (
  cd "${PROJECT}"
  mvn_log="${BASE_DIR}/mvn_deploy_release.log"
  log "Staging release in nexus"
  maven_deploy release "$mvn_log"
  declare staged_repo_id
  declare hadoop3_staged_repo_id
  if ! is_dry_run; then
    mapfile -t staged_repo_ids < <(grep -o "Closing staging repository with ID .*" "$mvn_log" \
        | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/')
    log "Release artifacts successfully published to repo: " "${staged_repo_ids[@]}"
    repo_count="${#staged_repo_ids[@]}"
    if [[ "${repo_count}" == "2" ]]; then
      staged_repo_id=${staged_repo_ids[0]}
      hadoop3_staged_repo_id=${staged_repo_ids[1]}
    elif [[ "${repo_count}" == "1" ]]; then
      staged_repo_id=${staged_repo_ids[0]}
      hadoop3_staged_repo_id="not-applicable"
    else
      staged_repo_id="not-applicable"
      hadoop3_staged_repo_id="not-applicable"
    fi
    rm "$mvn_log"
  else
    log "Dry run: Release artifacts successfully built, but not published due to dry run."
    staged_repo_id="dryrun-no-repo"
    hadoop3_staged_repo_id="dryrun-no-repo"
  fi
  export staged_repo_id
  export hadoop3_staged_repo_id
  # Dump out email to send. Where we find vote.tmpl depends
  # on where this script is run from
  PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g'
  export PROJECT_TEXT
  eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt"
  )
  exit $?
fi

set +x  # done with detailed logging
cd ..
rm -rf "${PROJECT}"
log "ERROR: expects to be called with 'tag', 'publish-dist', 'publish-release', or 'publish-snapshot'" >&2
exit_with_usage