Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26362 Upload mvn site artifacts for nightly build to nightlies (#3778) 

Signed-off-by: Yulin Niu <niuyulin@apache.org>
This commit is contained in:
Duo Zhang 2021-10-21 23:07:30 +08:00 committed by GitHub
parent 4838246ba3
commit 69656e0df8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 82 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
dev-support/Jenkinsfile vendored
View File

@ -54,6 +54,8 @@ pipeline {
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check' SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'
DEEP_CHECKS = 'compile,htmlout,javac,maven,mvninstall,shadedjars,unit' // run by 'yetus jdkX (HadoopY) checks' DEEP_CHECKS = 'compile,htmlout,javac,maven,mvninstall,shadedjars,unit' // run by 'yetus jdkX (HadoopY) checks'
ASF_NIGHTLIES = 'https://nightlies.apache.org' ASF_NIGHTLIES = 'https://nightlies.apache.org'
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
} }
parameters { parameters {
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
@ -205,6 +207,7 @@ pipeline {
SET_JAVA_HOME = '/usr/lib/jvm/java-8' SET_JAVA_HOME = '/usr/lib/jvm/java-8'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}" OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}" OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}"
} }
steps { steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile // Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@ -253,6 +256,31 @@ pipeline {
post { post {
always { always {
stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
transfers: [
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/*-site/*,${env.OUTPUT_DIR_RELATIVE}/*-site/**/*"
)
]
)
])
sh '''#!/bin/bash -e
if [ -d "${OUTPUT_DIR}/branch-site" ]; then
echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
rm -rf "${OUTPUT_DIR}/branch-site"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
else
echo "No branch-site, skipping"
fi
if [ -d "${OUTPUT_DIR}/patch-site" ]; then
echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
rm -rf "${OUTPUT_DIR}/patch-site"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
else
echo "No patch-site, skipping"
fi
'''
// Has to be relative to WORKSPACE. // Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
@ -358,7 +386,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip" rm -rf "${OUTPUT_DIR}/test_logs.zip"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi
@ -468,7 +496,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip" rm -rf "${OUTPUT_DIR}/test_logs.zip"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi
@ -582,7 +610,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip" rm -rf "${OUTPUT_DIR}/test_logs.zip"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi
@ -698,7 +726,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip" rm -rf "${OUTPUT_DIR}/test_logs.zip"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi

36
dev-support/Jenkinsfile_GitHub
View File

@ -53,6 +53,8 @@ pipeline {
WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check' WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check' WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
ASF_NIGHTLIES = 'https://nightlies.apache.org' ASF_NIGHTLIES = 'https://nightlies.apache.org'
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
} }
parameters { parameters {
@ -83,6 +85,7 @@ pipeline {
BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}" BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}" DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}" YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}"
} }
steps { steps {
dir("${SOURCEDIR}") { dir("${SOURCEDIR}") {
@ -124,9 +127,34 @@ pipeline {
} }
post { post {
always { always {
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
transfers: [
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/*,${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/**/*"
)
]
)
])
sh '''#!/bin/bash -e
if [ -d "${PATCHDIR}/branch-site" ]; then
echo "Remove ${PATCHDIR}/branch-site for saving space"
rm -rf "${PATCHDIR}/branch-site"
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
else
echo "No branch-site, skipping"
fi
if [ -d "${PATCHDIR}/patch-site" ]; then
echo "Remove ${PATCHDIR}/patch-site for saving space"
rm -rf "${PATCHDIR}/patch-site"
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
else
echo "No patch-site, skipping"
fi
'''
// Has to be relative to WORKSPACE. // Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit" archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*"
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*" archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*"
publishHTML target: [ publishHTML target: [
allowMissing: true, allowMissing: true,
keepAll: true, keepAll: true,
@ -250,7 +278,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space" echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip" rm -rf "${PATCHDIR}/test_logs.zip"
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi
@ -381,7 +409,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space" echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip" rm -rf "${PATCHDIR}/test_logs.zip"
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi

10
dev-support/hbase-personality.sh
View File

@ -115,6 +115,10 @@ function personality_parse_args
delete_parameter "${i}" delete_parameter "${i}"
SKIP_ERRORPRONE=true SKIP_ERRORPRONE=true
;; ;;
--asf-nightlies-general-check-base=*)
delete_parameter "${i}"
ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
;;
esac esac
done done
} }
@ -414,7 +418,11 @@ function refguide_rebuild
fi fi
add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect." add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html" if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
else
add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
fi
return 0 return 0
} }

5
dev-support/hbase_nightly_yetus.sh
View File

@ -96,6 +96,11 @@ if [[ ! -d "${OUTPUT_DIR}" ]]; then
exit 1 exit 1
fi fi
# pass asf nightlies url in
if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
YETUS_ARGS=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" "${YETUS_ARGS[@]}")
fi
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}") YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch" TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"

4
dev-support/jenkins_precommit_github_yetus.sh
View File

@ -143,6 +143,10 @@ fi
YETUS_ARGS+=("--sentinel") YETUS_ARGS+=("--sentinel")
# use emoji vote so it is easier to find the broken line # use emoji vote so it is easier to find the broken line
YETUS_ARGS+=("--github-use-emoji-vote") YETUS_ARGS+=("--github-use-emoji-vote")
# pass asf nightlies url in
if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}")
fi
echo "Launching yetus with command line:" echo "Launching yetus with command line:"
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"