diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 5af360aa35b..53ad05020f3 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -50,6 +50,8 @@ pipeline { TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite,xml' EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes" ASF_NIGHTLIES = 'https://nightlies.apache.org' + ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" + ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. @@ -141,6 +143,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}" OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}" SKIP_ERRORPRONE = 'true' + ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" } steps { // Must do prior to anything else, since if one of them timesout we'll stash the commentfile @@ -188,6 +191,31 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" post { always { stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/*-site/*,${env.OUTPUT_DIR_RELATIVE}/*-site/**/*" + ) + ] + ) + ]) + sh '''#!/bin/bash -e + if [ -d "${OUTPUT_DIR}/branch-site" ]; then + echo "Remove ${OUTPUT_DIR}/branch-site for saving space" + rm -rf "${OUTPUT_DIR}/branch-site" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html" + else + echo "No branch-site, skipping" + fi + if [ -d "${OUTPUT_DIR}/patch-site" ]; then + echo "Remove ${OUTPUT_DIR}/patch-site for saving space" + rm -rf "${OUTPUT_DIR}/patch-site" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html" + else + echo "No patch-site, skipping" + fi + ''' // Has to be relative to WORKSPACE. archiveArtifacts "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts "${env.OUTPUT_DIR_RELATIVE}/**/*" @@ -294,7 +322,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -402,7 +430,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -517,7 +545,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 65988e70f39..7e9a8002962 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -118,6 +118,10 @@ function personality_parse_args delete_parameter "${i}" SKIP_ERRORPRONE=true ;; + --asf-nightlies-general-check-base=*) + delete_parameter "${i}" + ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=} + ;; esac done } @@ -399,7 +403,11 @@ function refguide_rebuild fi add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect." - add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html" + if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then + add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html" + else + add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html" + fi return 0 } diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index 2ec03c5ebdb..78403775def 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -98,6 +98,11 @@ if [[ ! -d "${OUTPUT_DIR}" ]]; then exit 1 fi +# pass asf nightlies url in +if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then + YETUS_ARGS=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" "${YETUS_ARGS[@]}") +fi + if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}") TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"