diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 95659b89149..d9871820acd 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -54,6 +54,8 @@ pipeline { SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check' DEEP_CHECKS = 'compile,htmlout,javac,maven,mvninstall,shadedjars,unit' // run by 'yetus jdkX (HadoopY) checks' ASF_NIGHTLIES = 'https://nightlies.apache.org' + ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" + ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. @@ -205,6 +207,7 @@ pipeline { SET_JAVA_HOME = '/usr/lib/jvm/java-8' OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}" OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}" + ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" } steps { // Must do prior to anything else, since if one of them timesout we'll stash the commentfile @@ -253,6 +256,31 @@ pipeline { post { always { stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/*-site/*,${env.OUTPUT_DIR_RELATIVE}/*-site/**/*" + ) + ] + ) + ]) + sh '''#!/bin/bash -e + if [ -d "${OUTPUT_DIR}/branch-site" ]; then + echo "Remove ${OUTPUT_DIR}/branch-site for saving space" + rm -rf "${OUTPUT_DIR}/branch-site" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html" + else + echo "No branch-site, skipping" + fi + if [ -d "${OUTPUT_DIR}/patch-site" ]; then + echo "Remove ${OUTPUT_DIR}/patch-site for saving space" + rm -rf "${OUTPUT_DIR}/patch-site" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html" + else + echo "No patch-site, skipping" + fi + ''' // Has to be relative to WORKSPACE. archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" @@ -358,7 +386,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -468,7 +496,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -582,7 +610,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -698,7 +726,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index 530fc6d6005..c9dee54ae64 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -53,6 +53,8 @@ pipeline { WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check' WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check' ASF_NIGHTLIES = 'https://nightlies.apache.org' + ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" + ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" } parameters { @@ -83,6 +85,7 @@ pipeline { BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}" DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}" YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}" + ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" } steps { dir("${SOURCEDIR}") { @@ -124,9 +127,34 @@ pipeline { } post { always { + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/*,${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/**/*" + ) + ] + ) + ]) + sh '''#!/bin/bash -e + if [ -d "${PATCHDIR}/branch-site" ]; then + echo "Remove ${PATCHDIR}/branch-site for saving space" + rm -rf "${PATCHDIR}/branch-site" + python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html" + else + echo "No branch-site, skipping" + fi + if [ -d "${PATCHDIR}/patch-site" ]; then + echo "Remove ${PATCHDIR}/patch-site for saving space" + rm -rf "${PATCHDIR}/patch-site" + python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html" + else + echo "No patch-site, skipping" + fi + ''' // Has to be relative to WORKSPACE. - archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit" - archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*" + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*" + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*" publishHTML target: [ allowMissing: true, keepAll: true, @@ -250,7 +278,7 @@ pipeline { if [ -f "${PATCHDIR}/test_logs.zip" ]; then echo "Remove ${PATCHDIR}/test_logs.zip for saving space" rm -rf "${PATCHDIR}/test_logs.zip" - python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" + python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -381,7 +409,7 @@ pipeline { if [ -f "${PATCHDIR}/test_logs.zip" ]; then echo "Remove ${PATCHDIR}/test_logs.zip for saving space" rm -rf "${PATCHDIR}/test_logs.zip" - python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" + python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 6eb38e34228..6de1e06b4fb 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -115,6 +115,10 @@ function personality_parse_args delete_parameter "${i}" SKIP_ERRORPRONE=true ;; + --asf-nightlies-general-check-base=*) + delete_parameter "${i}" + ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=} + ;; esac done } @@ -414,7 +418,11 @@ function refguide_rebuild fi add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect." - add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html" + if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then + add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html" + else + add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html" + fi return 0 } diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index 65b5270f0bf..3fb1ac7bfce 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -96,6 +96,11 @@ if [[ ! -d "${OUTPUT_DIR}" ]]; then exit 1 fi +# pass asf nightlies url in +if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then + YETUS_ARGS=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" "${YETUS_ARGS[@]}") +fi + if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}") TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch" diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index 5bb2b1b755a..9652d1e471f 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -143,6 +143,10 @@ fi YETUS_ARGS+=("--sentinel") # use emoji vote so it is easier to find the broken line YETUS_ARGS+=("--github-use-emoji-vote") +# pass asf nightlies url in +if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then + YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}") +fi echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"