HBASE-26362 Upload mvn site artifacts for nightly build to nightlies (#3778)
Signed-off-by: Yulin Niu <niuyulin@apache.org>
This commit is contained in:
parent
4838246ba3
commit
69656e0df8
|
@ -54,6 +54,8 @@ pipeline {
|
|||
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'
|
||||
DEEP_CHECKS = 'compile,htmlout,javac,maven,mvninstall,shadedjars,unit' // run by 'yetus jdkX (HadoopY) checks'
|
||||
ASF_NIGHTLIES = 'https://nightlies.apache.org'
|
||||
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
|
||||
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
|
||||
}
|
||||
parameters {
|
||||
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
|
||||
|
@ -205,6 +207,7 @@ pipeline {
|
|||
SET_JAVA_HOME = '/usr/lib/jvm/java-8'
|
||||
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
|
||||
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
|
||||
ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}"
|
||||
}
|
||||
steps {
|
||||
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
|
||||
|
@ -253,6 +256,31 @@ pipeline {
|
|||
post {
|
||||
always {
|
||||
stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
|
||||
sshPublisher(publishers: [
|
||||
sshPublisherDesc(configName: 'Nightlies',
|
||||
transfers: [
|
||||
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
|
||||
sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/*-site/*,${env.OUTPUT_DIR_RELATIVE}/*-site/**/*"
|
||||
)
|
||||
]
|
||||
)
|
||||
])
|
||||
sh '''#!/bin/bash -e
|
||||
if [ -d "${OUTPUT_DIR}/branch-site" ]; then
|
||||
echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
|
||||
rm -rf "${OUTPUT_DIR}/branch-site"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
|
||||
else
|
||||
echo "No branch-site, skipping"
|
||||
fi
|
||||
if [ -d "${OUTPUT_DIR}/patch-site" ]; then
|
||||
echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
|
||||
rm -rf "${OUTPUT_DIR}/patch-site"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
|
||||
else
|
||||
echo "No patch-site, skipping"
|
||||
fi
|
||||
'''
|
||||
// Has to be relative to WORKSPACE.
|
||||
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
|
||||
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
|
||||
|
@ -358,7 +386,7 @@ pipeline {
|
|||
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
|
||||
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
|
||||
rm -rf "${OUTPUT_DIR}/test_logs.zip"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
else
|
||||
echo "No test_logs.zip, skipping"
|
||||
fi
|
||||
|
@ -468,7 +496,7 @@ pipeline {
|
|||
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
|
||||
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
|
||||
rm -rf "${OUTPUT_DIR}/test_logs.zip"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
else
|
||||
echo "No test_logs.zip, skipping"
|
||||
fi
|
||||
|
@ -582,7 +610,7 @@ pipeline {
|
|||
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
|
||||
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
|
||||
rm -rf "${OUTPUT_DIR}/test_logs.zip"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
else
|
||||
echo "No test_logs.zip, skipping"
|
||||
fi
|
||||
|
@ -698,7 +726,7 @@ pipeline {
|
|||
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
|
||||
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
|
||||
rm -rf "${OUTPUT_DIR}/test_logs.zip"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
|
||||
else
|
||||
echo "No test_logs.zip, skipping"
|
||||
fi
|
||||
|
|
|
@ -53,6 +53,8 @@ pipeline {
|
|||
WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
|
||||
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
|
||||
ASF_NIGHTLIES = 'https://nightlies.apache.org'
|
||||
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
|
||||
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
|
||||
}
|
||||
|
||||
parameters {
|
||||
|
@ -83,6 +85,7 @@ pipeline {
|
|||
BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
|
||||
DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
|
||||
YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
|
||||
ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}"
|
||||
}
|
||||
steps {
|
||||
dir("${SOURCEDIR}") {
|
||||
|
@ -124,9 +127,34 @@ pipeline {
|
|||
}
|
||||
post {
|
||||
always {
|
||||
sshPublisher(publishers: [
|
||||
sshPublisherDesc(configName: 'Nightlies',
|
||||
transfers: [
|
||||
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
|
||||
sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/*,${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/**/*"
|
||||
)
|
||||
]
|
||||
)
|
||||
])
|
||||
sh '''#!/bin/bash -e
|
||||
if [ -d "${PATCHDIR}/branch-site" ]; then
|
||||
echo "Remove ${PATCHDIR}/branch-site for saving space"
|
||||
rm -rf "${PATCHDIR}/branch-site"
|
||||
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
|
||||
else
|
||||
echo "No branch-site, skipping"
|
||||
fi
|
||||
if [ -d "${PATCHDIR}/patch-site" ]; then
|
||||
echo "Remove ${PATCHDIR}/patch-site for saving space"
|
||||
rm -rf "${PATCHDIR}/patch-site"
|
||||
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
|
||||
else
|
||||
echo "No patch-site, skipping"
|
||||
fi
|
||||
'''
|
||||
// Has to be relative to WORKSPACE.
|
||||
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
|
||||
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
|
||||
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*"
|
||||
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*"
|
||||
publishHTML target: [
|
||||
allowMissing: true,
|
||||
keepAll: true,
|
||||
|
@ -250,7 +278,7 @@ pipeline {
|
|||
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
|
||||
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
|
||||
rm -rf "${PATCHDIR}/test_logs.zip"
|
||||
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
|
||||
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
|
||||
else
|
||||
echo "No test_logs.zip, skipping"
|
||||
fi
|
||||
|
@ -381,7 +409,7 @@ pipeline {
|
|||
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
|
||||
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
|
||||
rm -rf "${PATCHDIR}/test_logs.zip"
|
||||
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
|
||||
python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
|
||||
else
|
||||
echo "No test_logs.zip, skipping"
|
||||
fi
|
||||
|
|
|
@ -115,6 +115,10 @@ function personality_parse_args
|
|||
delete_parameter "${i}"
|
||||
SKIP_ERRORPRONE=true
|
||||
;;
|
||||
--asf-nightlies-general-check-base=*)
|
||||
delete_parameter "${i}"
|
||||
ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
@ -414,7 +418,11 @@ function refguide_rebuild
|
|||
fi
|
||||
|
||||
add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
|
||||
add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
|
||||
if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
|
||||
add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
|
||||
else
|
||||
add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
|
|
|
@ -96,6 +96,11 @@ if [[ ! -d "${OUTPUT_DIR}" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# pass asf nightlies url in
|
||||
if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
|
||||
YETUS_ARGS=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" "${YETUS_ARGS[@]}")
|
||||
fi
|
||||
|
||||
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
|
||||
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
|
||||
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
|
||||
|
|
|
@ -143,6 +143,10 @@ fi
|
|||
YETUS_ARGS+=("--sentinel")
|
||||
# use emoji vote so it is easier to find the broken line
|
||||
YETUS_ARGS+=("--github-use-emoji-vote")
|
||||
# pass asf nightlies url in
|
||||
if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
|
||||
YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}")
|
||||
fi
|
||||
|
||||
echo "Launching yetus with command line:"
|
||||
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
|
||||
|
|
Loading…
Reference in New Issue