HBASE-27293 Remove jenkins and personality scripts support for 1.x (#4690)

Signed-off-by: GeorryHuang <huangzhuoyue@apache.org>
(cherry picked from commit bffae99f40)
This commit is contained in:
Duo Zhang 2022-08-11 16:31:30 +08:00
parent 2941b8e583
commit 4079647c39
2 changed files with 71 additions and 265 deletions

View File

@ -34,7 +34,6 @@ pipeline {
YETUS_RELEASE = '0.12.0'
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7'
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3'
OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
@ -186,7 +185,6 @@ pipeline {
// stash with given name for all tests we might run, so that we can unstash all of them even if
// we skip some due to e.g. branch-specific JDK or Hadoop support
stash name: 'general-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_GENERAL}/doesn't-match"
stash name: 'jdk7-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK7}/doesn't-match"
stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match"
stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match"
stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
@ -296,121 +294,14 @@ pipeline {
}
}
}
stage ('yetus jdk7 checks') {
stage ('yetus jdk8 hadoop2 checks') {
agent {
node {
label 'hbase'
}
}
when {
branch 'branch-1*'
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
SET_JAVA_HOME = "/usr/lib/jvm/java-7"
}
steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
sh '''#!/usr/bin/env bash
set -e
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
echo '(x) {color:red}-1 jdk7 checks{color}' >"${OUTPUT_DIR}/commentfile"
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
'''
unstash 'yetus'
dir('component') {
checkout scm
}
sh '''#!/usr/bin/env bash
set -e
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
script {
def ret = sh(
returnStatus: true,
script: '''#!/usr/bin/env bash
set -e
declare -i status=0
if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
echo '(/) {color:green}+1 jdk7 checks{color}' > "${OUTPUT_DIR}/commentfile"
else
echo '(x) {color:red}-1 jdk7 checks{color}' > "${OUTPUT_DIR}/commentfile"
status=1
fi
echo "-- For more information [see jdk7 report|${BUILD_URL}/JDK7_20Nightly_20Build_20Report/]" >> "${OUTPUT_DIR}/commentfile"
exit "${status}"
'''
)
if (ret != 0) {
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
// test output. See HBASE-26339 for more details.
currentBuild.result = 'UNSTABLE'
}
}
}
post {
always {
stash name: 'jdk7-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
// zip surefire reports.
sh '''#!/bin/bash -e
if [ -d "${OUTPUT_DIR}/archiver" ]; then
count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
if [[ 0 -ne ${count} ]]; then
echo "zipping ${count} archived files"
zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
else
echo "No archived files, skipping compressing."
fi
else
echo "No archiver directory, skipping compressing."
fi
'''
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
transfers: [
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip"
)
]
)
])
// remove the big test logs zip file, store the nightlies url in test_logs.html
sh '''#!/bin/bash -e
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
'''
// Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
publishHTML target: [
allowMissing : true,
keepAll : true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE.
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
reportFiles : 'console-report.html',
reportName : 'JDK7 Nightly Build Report'
]
}
}
}
stage ('yetus jdk8 hadoop2 checks') {
agent {
node {
label 'hbase'
}
branch 'branch-2*'
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
@ -519,11 +410,6 @@ pipeline {
label 'hbase'
}
}
when {
not {
branch 'branch-1*'
}
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"
@ -633,11 +519,6 @@ pipeline {
label 'hbase'
}
}
when {
not {
branch 'branch-1*'
}
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"
@ -814,7 +695,7 @@ pipeline {
'''
unstash 'hadoop-2'
sh '''#!/bin/bash -xe
if [[ "${BRANCH}" = branch-2* ]] || [[ "${BRANCH}" = branch-1* ]]; then
if [[ "${BRANCH}" = branch-2* ]]; then
echo "Attempting to use run an instance on top of Hadoop 2."
artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
@ -838,44 +719,40 @@ pipeline {
'''
unstash 'hadoop-3'
sh '''#!/bin/bash -e
if [[ "${BRANCH}" = branch-1* ]]; then
echo "Skipping to run against Hadoop 3 for branch ${BRANCH}"
else
echo "Attempting to use run an instance on top of Hadoop 3."
artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
--single-process \
--working-dir output-integration/hadoop-3 \
--hbase-client-install hbase-client \
hbase-install \
hadoop-3/bin/hadoop \
hadoop-3/share/hadoop/yarn/timelineservice \
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
>output-integration/hadoop-3.log 2>&1 ; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
exit 2
fi
echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
--single-process \
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
--working-dir output-integration/hadoop-3-shaded \
--hbase-client-install hbase-client \
hbase-install \
hadoop-3/bin/hadoop \
hadoop-3/share/hadoop/yarn/timelineservice \
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
>output-integration/hadoop-3-shaded.log 2>&1 ; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
exit 2
fi
echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
echo "Attempting to use run an instance on top of Hadoop 3."
artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
--single-process \
--working-dir output-integration/hadoop-3 \
--hbase-client-install hbase-client \
hbase-install \
hadoop-3/bin/hadoop \
hadoop-3/share/hadoop/yarn/timelineservice \
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
>output-integration/hadoop-3.log 2>&1 ; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
exit 2
fi
echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
--single-process \
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
--working-dir output-integration/hadoop-3-shaded \
--hbase-client-install hbase-client \
hbase-install \
hadoop-3/bin/hadoop \
hadoop-3/share/hadoop/yarn/timelineservice \
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
>output-integration/hadoop-3-shaded.log 2>&1 ; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
exit 2
fi
echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
'''
}
post {
@ -916,14 +793,12 @@ pipeline {
script {
try {
unstash 'general-result'
unstash 'jdk7-result'
unstash 'jdk8-hadoop2-result'
unstash 'jdk8-hadoop3-result'
unstash 'jdk11-hadoop3-result'
unstash 'srctarball-result'
sh "printenv"
def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile",
@ -948,8 +823,14 @@ pipeline {
echo "[INFO] Comment:"
echo comment
echo ""
echo "[INFO] There are ${currentBuild.changeSets.size()} change sets."
getJirasToComment(currentBuild).each { currentIssue ->
echo "[DEBUG] checking to see if feature branch"
def jiras = getJirasToComment(env.BRANCH_NAME, [])
if (jiras.isEmpty()) {
echo "[DEBUG] non-feature branch, checking change messages for jira keys."
echo "[INFO] There are ${currentBuild.changeSets.size()} change sets."
jiras = getJirasToCommentFromChangesets(currentBuild)
}
jiras.each { currentIssue ->
jiraComment issueKey: currentIssue, body: comment
}
} catch (Exception exception) {
@ -962,7 +843,7 @@ pipeline {
}
import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper
@NonCPS
List<String> getJirasToComment(RunWrapper thisBuild) {
List<String> getJirasToCommentFromChangesets(RunWrapper thisBuild) {
def seenJiras = []
thisBuild.changeSets.each { cs ->
cs.getItems().each { change ->
@ -972,16 +853,21 @@ List<String> getJirasToComment(RunWrapper thisBuild) {
echo " ${change.commitId}"
echo " ${change.author}"
echo ""
msg.eachMatch("HBASE-[0-9]+") { currentIssue ->
echo "[DEBUG] found jira key: ${currentIssue}"
if (currentIssue in seenJiras) {
echo "[DEBUG] already commented on ${currentIssue}."
} else {
echo "[INFO] commenting on ${currentIssue}."
seenJiras << currentIssue
}
}
seenJiras = getJirasToComment(msg, seenJiras)
}
}
return seenJiras
}
@NonCPS
List<String> getJirasToComment(CharSequence source, List<String> seen) {
source.eachMatch("HBASE-[0-9]+") { currentIssue ->
echo "[DEBUG] found jira key: ${currentIssue}"
if (currentIssue in seen) {
echo "[DEBUG] already commented on ${currentIssue}."
} else {
echo "[INFO] commenting on ${currentIssue}."
seen << currentIssue
}
}
return seen
}

View File

@ -80,9 +80,9 @@ function personality_globals
# TODO use PATCH_BRANCH to select jdk versions to use.
# Yetus 0.7.0 enforces limits. Default proclimit is 1000.
# Up it. See HBASE-19902 for how we arrived at this number.
# Up it. See HBASE-25081 for how we arrived at this number.
#shellcheck disable=SC2034
PROC_LIMIT=12500
PROC_LIMIT=30000
# Set docker container to run with 20g. Default is 4g in yetus.
# See HBASE-19902 for how we arrived at 20g.
@ -145,8 +145,6 @@ function personality_modules
local repostatus=$1
local testtype=$2
local extra=""
local branch1jdk8=()
local jdk8module=""
local MODULES=("${CHANGED_MODULES[@]}")
yetus_info "Personality: ${repostatus} ${testtype}"
@ -173,10 +171,6 @@ function personality_modules
tmpdir=$(realpath target)
extra="${extra} -Djava.io.tmpdir=${tmpdir} -DHBasePatchProcess"
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
extra="${extra} -Dhttps.protocols=TLSv1.2"
fi
# If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along
# the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3
# logic is not both activated within Maven.
@ -207,21 +201,6 @@ function personality_modules
return
fi
# This list should include any modules that require jdk8. Maven should be configured to only
# include them when a proper JDK is in use, but that doesn' work if we specifically ask for the
# module to build as yetus does if something changes in the module. Rather than try to
# figure out what jdk is in use so we can duplicate the module activation logic, just
# build at the top level if anything changes in one of these modules and let maven sort it out.
branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache)
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
for jdk8module in "${branch1jdk8[@]}"; do
if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then
MODULES=(.)
break
fi
done
fi
if [[ ${testtype} == spotbugs ]]; then
# Run spotbugs on each module individually to diff pre-patch and post-patch results and
# report new warnings for changed modules only.
@ -241,8 +220,7 @@ function personality_modules
return
fi
if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
[[ "${PATCH_BRANCH}" != branch-1* ]] ; then
if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]]; then
extra="${extra} -PerrorProne"
fi
@ -445,11 +423,7 @@ function refguide_rebuild
return 1
fi
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
pdf_output="book.pdf"
else
pdf_output="apache_hbase_reference_guide.pdf"
fi
pdf_output="apache_hbase_reference_guide.pdf"
if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then
add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide."
@ -601,76 +575,22 @@ function hadoopcheck_rebuild
# All supported Hadoop versions that we want to test the compilation with
# See the Hadoop section on prereqs in the HBase Reference Guide
if [[ "${PATCH_BRANCH}" = branch-1.3 ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-1.3 rules."
if [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-2.4+ rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7"
hbase_hadoop2_versions="2.10.1"
else
hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
fi
elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.7.7"
else
hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7"
fi
elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.10.0"
else
hbase_hadoop2_versions="2.10.0"
fi
elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5"
else
hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
fi
elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.7.7 2.8.5"
else
hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5"
fi
elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
else
hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
fi
elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.10.0"
else
hbase_hadoop2_versions="2.10.0"
hbase_hadoop2_versions="2.10.0 2.10.1"
fi
else
yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
hbase_hadoop2_versions=""
fi
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
hbase_hadoop3_versions=""
elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then
yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules"
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop3_versions="3.0.3 3.1.2"
else
hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2"
fi
yetus_info "Setting Hadoop 3 versions to test based on branch-2.4+/master/feature branch rules"
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop3_versions="3.1.2 3.2.2 3.3.1"
else
yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules"
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop3_versions="3.1.2 3.2.1"
else
hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1"
fi
hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2 3.3.0 3.3.1"
fi
export MAVEN_OPTS="${MAVEN_OPTS}"