HBASE-26362 Upload mvn site artifacts for nightly build to nightlies (#3778)

Signed-off-by: Yulin Niu <niuyulin@apache.org>

dev-support/Jenkinsfile

@@ -50,6 +50,8 @@ pipeline {
     TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite,xml'
     EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
     ASF_NIGHTLIES = 'https://nightlies.apache.org'
+    ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
+    ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
   }
   parameters {
     booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
@@ -141,6 +143,7 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
             OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
             OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
             SKIP_ERRORPRONE = 'true'
+            ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}"
           }
           steps {
             // Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@@ -188,6 +191,31 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
           post {
             always {
               stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+              sshPublisher(publishers: [
+                sshPublisherDesc(configName: 'Nightlies',
+                  transfers: [
+                    sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
+                      sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/*-site/*,${env.OUTPUT_DIR_RELATIVE}/*-site/**/*"
+                    )
+                  ]
+                )
+              ])
+              sh '''#!/bin/bash -e
+              if [ -d "${OUTPUT_DIR}/branch-site" ]; then
+                echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
+                rm -rf "${OUTPUT_DIR}/branch-site"
+                python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
+              else
+                echo "No branch-site, skipping"
+              fi
+              if [ -d "${OUTPUT_DIR}/patch-site" ]; then
+                echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
+                rm -rf "${OUTPUT_DIR}/patch-site"
+                python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
+              else
+                echo "No patch-site, skipping"
+              fi
+              '''
               // Has to be relative to WORKSPACE.
               archiveArtifacts "${env.OUTPUT_DIR_RELATIVE}/*"
               archiveArtifacts "${env.OUTPUT_DIR_RELATIVE}/**/*"
@@ -294,7 +322,7 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -402,7 +430,7 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -517,7 +545,7 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi

dev-support/hbase-personality.sh

@@ -118,6 +118,10 @@ function personality_parse_args
         delete_parameter "${i}"
         SKIP_ERRORPRONE=true
       ;;
+      --asf-nightlies-general-check-base=*)
+        delete_parameter "${i}"
+        ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
+      ;;
     esac
   done
 }
@@ -399,7 +403,11 @@ function refguide_rebuild
   fi
 
   add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
-  add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
+  if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
+    add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
+  else
+    add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
+  fi
   return 0
 }
 

dev-support/hbase_nightly_yetus.sh

@@ -98,6 +98,11 @@ if [[ ! -d "${OUTPUT_DIR}" ]]; then
   exit 1
 fi
 
+# pass asf nightlies url in
+if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
+  YETUS_ARGS=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" "${YETUS_ARGS[@]}")
+fi
+
 if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
   YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
   TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"