HBASE-27449 Use xmllint to parse version and revision in release scripts (#4850)
Also upgrade our Dockerfile for release scripts to use ubuntu 22.04, maven 3.8.6 and yetus 0.14.1. Use python3 only. Signed-off-by: Guanghao Zhang <zghao@apache.org> Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>
This commit is contained in:
Duo Zhang
GitHub
parent
f09ac40105
commit
984d226010
|
|
@ -15,11 +15,11 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Image for building HBase releases. Based on Ubuntu 18.04.
|
||||
# Image for building HBase releases. Based on Ubuntu 22.04.
|
||||
#
|
||||
# Includes:
|
||||
# * Java 8
|
||||
FROM ubuntu:18.04
|
||||
FROM ubuntu:22.04
|
||||
|
||||
|
||||
# Install extra needed repos and refresh.
|
||||
|
|
@ -28,40 +28,46 @@ FROM ubuntu:18.04
|
|||
# the most current package versions (instead of potentially using old versions cached by docker).
|
||||
RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
|
||||
curl='7.58.0-*' \
|
||||
git='1:2.17.1-*' \
|
||||
gnupg='2.2.4-*' \
|
||||
libcurl4-openssl-dev='7.58.0-*' \
|
||||
libxml2-dev='2.9.4+dfsg1-*' \
|
||||
lsof='4.89+dfsg-*' \
|
||||
curl='7.81.0-*' \
|
||||
git='1:2.34.1-*' \
|
||||
gnupg='2.2.27-*' \
|
||||
libcurl4-openssl-dev='7.81.0-*' \
|
||||
libxml2-dev='2.9.13+dfsg-*' \
|
||||
libxml2-utils='2.9.13+dfsg-*' \
|
||||
lsof='4.93.2+dfsg-*' \
|
||||
openjdk-8-jdk='8u*' \
|
||||
python-pip='9.0.1-*' \
|
||||
subversion='1.9.7-*' \
|
||||
wget='1.19.4-*' \
|
||||
python3-pip='22.0.2+dfsg-*' \
|
||||
subversion='1.14.1-*' \
|
||||
wget='1.21.2-*' \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
|
||||
&& pip install \
|
||||
python-dateutil==2.8.1
|
||||
&& pip3 install \
|
||||
python-dateutil==2.8.2
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
# Install mvn 3.6.3.
|
||||
ARG MAVEN_VERSION=3.6.3
|
||||
ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0
|
||||
ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
|
||||
# Install maven
|
||||
ENV MAVEN_VERSION=3.8.6
|
||||
ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
|
||||
ARG MAVEN_SHA512="f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26"
|
||||
RUN mkdir -p /opt/maven \
|
||||
&& curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
|
||||
&& echo "${SHA} /tmp/apache-maven.tar.gz" | sha512sum -c - \
|
||||
&& curl -fsSL -o /tmp/apache-maven.tar.gz "${MAVEN_URL}" \
|
||||
&& echo "${MAVEN_SHA512} /tmp/apache-maven.tar.gz" | sha512sum -c - \
|
||||
&& tar -xzf /tmp/apache-maven.tar.gz -C /opt/maven --strip-components=1 \
|
||||
&& rm -f /tmp/apache-maven.tar.gz \
|
||||
&& ln -s /opt/maven/bin/mvn /usr/bin/mvn
|
||||
|
||||
# Install Apache Yetus
|
||||
ENV YETUS_VERSION 0.12.0
|
||||
RUN curl -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \
|
||||
tar xvz -C /opt
|
||||
ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
|
||||
ENV YETUS_VERSION 0.14.1
|
||||
ARG YETUS_URL="https://archive.apache.org/dist/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz"
|
||||
ARG YETUS_SHA512="068c8426a5c7ecc9281b988123d223a3d772fd725365303bde8287ae1515e623764b89f09a4a9cdc01e7f30631814086b71408b625025a51bdb3c58b213b4109"
|
||||
RUN mkdir -p /opt/yetus \
|
||||
&& curl -fsSL -o /tmp/apache-yetus.tar.gz "${YETUS_URL}" \
|
||||
&& echo "${YETUS_SHA512} */tmp/apache-yetus.tar.gz" | sha512sum -c - \
|
||||
&& tar -xzf /tmp/apache-yetus.tar.gz -C /opt/yetus --strip-components=1 \
|
||||
&& rm -f /tmp/apache-yetus.tar.gz
|
||||
ENV YETUS_HOME /opt/yetus
|
||||
|
||||
ARG UID
|
||||
ARG RM_USER
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ if len(sys.argv) != 3:
|
|||
print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
|
||||
sys.exit(1)
|
||||
|
||||
pattern = re.compile(r'^## Release .+ - Unreleased .+$')
|
||||
pattern = re.compile(r'^## Release .+$')
|
||||
with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
|
||||
line = prev_r.readline()
|
||||
while line:
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ if [[ "$1" == "tag" ]]; then
|
|||
# check their position when generating release data. We can not put them under the source code
|
||||
# directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
|
||||
# generating src release tarball, we will reset the git repo
|
||||
if [[ $(is_tracked "CHANGES.md") == 0 ]]; then
|
||||
if is_tracked "CHANGES.md"; then
|
||||
git add RELEASENOTES.md CHANGES.md
|
||||
git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
|
||||
cp CHANGES.md ../
|
||||
|
|
|
|||
|
|
@ -60,8 +60,11 @@ function read_config {
|
|||
}
|
||||
|
||||
function parse_version {
|
||||
grep -e '<version>.*</version>' | \
|
||||
head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1
|
||||
xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" -
|
||||
}
|
||||
|
||||
function parse_revision {
|
||||
xmllint --xpath "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='revision']/text()" -
|
||||
}
|
||||
|
||||
function banner {
|
||||
|
|
@ -135,6 +138,8 @@ function get_api_diff_version {
|
|||
# Get all branches that begin with 'branch-', the hbase convention for
|
||||
# release branches, sort them and then pop off the most recent.
|
||||
function get_release_info {
|
||||
init_xmllint
|
||||
|
||||
PROJECT="$(read_config "PROJECT" "$PROJECT")"
|
||||
export PROJECT
|
||||
|
||||
|
|
@ -164,6 +169,14 @@ function get_release_info {
|
|||
local version
|
||||
version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
|
||||
parse_version)"
|
||||
# We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
|
||||
# If we use ${revision} as placeholder, we need to parse the revision property to
|
||||
# get maven version
|
||||
# shellcheck disable=SC2016
|
||||
if [[ "${version}" == '${revision}' ]]; then
|
||||
version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
|
||||
parse_revision)"
|
||||
fi
|
||||
log "Current branch VERSION is $version."
|
||||
|
||||
NEXT_VERSION="$version"
|
||||
|
|
@ -344,6 +357,17 @@ function init_locale {
|
|||
export LANG="$locale_value"
|
||||
}
|
||||
|
||||
# Check whether xmllint is available
|
||||
function init_xmllint {
|
||||
if ! [ -x "$(command -v xmllint)" ]; then
|
||||
log "Error: xmllint is not available, we need to use it for parsing pom.xml." >&2
|
||||
log "Ubuntu: apt install libxml2-utils" >&2
|
||||
log "CentOS: yum install xmlstarlet" >&2
|
||||
log "Mac OS: brew install xmlstarlet" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Initializes JAVA_VERSION to the version of the JVM in use.
|
||||
function init_java {
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
|
|
@ -355,10 +379,10 @@ function init_java {
|
|||
}
|
||||
|
||||
function init_python {
|
||||
if ! [ -x "$(command -v python2)" ]; then
|
||||
error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2'
|
||||
if ! [ -x "$(command -v python3)" ]; then
|
||||
error 'python3 needed by yetus and api report. Install or add link?'
|
||||
fi
|
||||
log "python version: $(python2 --version)"
|
||||
log "python3 version: $(python3 --version)"
|
||||
}
|
||||
|
||||
# Set MVN
|
||||
|
|
@ -440,8 +464,8 @@ function git_clone_overwrite {
|
|||
log "Clone will be of the gitbox repo for ${PROJECT}."
|
||||
if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
|
||||
# Ugly!
|
||||
encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''', '')")
|
||||
encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''', '')")
|
||||
encoded_username=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_USERNAME''', ''))")
|
||||
encoded_password=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_PASSWORD''', ''))")
|
||||
GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
|
||||
else
|
||||
GIT_REPO="https://${asf_repo}"
|
||||
|
|
@ -533,7 +557,7 @@ function get_jira_name {
|
|||
# Update the CHANGES.md
|
||||
# DOES NOT DO COMMITS! Caller should do that.
|
||||
# requires yetus to have a defined home already.
|
||||
# yetus requires python2 to be on the path.
|
||||
# yetus requires python3 to be on the path.
|
||||
function update_releasenotes {
|
||||
local project_dir="$1"
|
||||
local jira_fix_version="$2"
|
||||
|
|
@ -686,28 +710,13 @@ function kick_gpg_agent {
|
|||
function maven_set_version { #input: <version_to_set>
|
||||
local this_version="$1"
|
||||
local use_revision='false'
|
||||
local -a version_splits=()
|
||||
IFS='.' read -ar version_splits <<< "$(maven_get_version)"
|
||||
|
||||
# Do the right thing based on project and release line.
|
||||
if [ "${PROJECT}" = 'hbase' ] ; then
|
||||
if [ "${version_splits[0]}" -le 1 ] ; then
|
||||
use_revision='false'
|
||||
elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -le 4 ] ; then
|
||||
use_revision='false'
|
||||
elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -ge 5 ] ; then
|
||||
use_revision='true'
|
||||
elif [ "${version_splits[0]}" -ge 3 ] ; then
|
||||
use_revision='true'
|
||||
fi
|
||||
elif [ "${PROJECT}" = 'hbase-thirdparty' ] ; then
|
||||
use_revision='false'
|
||||
elif [ "${PROJECT}" = 'hbase-connectors' ] ; then
|
||||
local maven_version
|
||||
maven_version="$(parse_version < pom.xml)"
|
||||
# We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
|
||||
# If we use ${revision} as placeholder, the way to bump maven version will be different
|
||||
# shellcheck disable=SC2016
|
||||
if [[ "${maven_version}" == '${revision}' ]]; then
|
||||
use_revision='true'
|
||||
elif [ "${PROJECT}" = 'hbase-filesystem' ] ; then
|
||||
use_revision='false'
|
||||
elif [ "${PROJECT}" = 'hbase-operator-tools' ] ; then
|
||||
use_revision='false'
|
||||
fi
|
||||
|
||||
if [ "${use_revision}" = 'false' ] ; then
|
||||
|
|
|
|||
Loading…
Reference in New Issue