Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26390 Upload src tarball to nightlies for nightly jobs (#3793) 

Signed-off-by: Xiaolin Ha <haxiaolin@apache.org>
This commit is contained in:
Duo Zhang 2021-10-25 23:32:15 +08:00
parent 444dec9289
commit 362117b52c
1 changed files with 32 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
dev-support/Jenkinsfile vendored
View File

@ -381,7 +381,7 @@ pipeline {
] ]
) )
]) ])
// remove the big test logs zip file, store the nightlies url in test_logs.txt // remove the big test logs zip file, store the nightlies url in test_logs.html
sh '''#!/bin/bash -e sh '''#!/bin/bash -e
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
@ -488,7 +488,7 @@ pipeline {
] ]
) )
]) ])
// remove the big test logs zip file, store the nightlies url in test_logs.txt // remove the big test logs zip file, store the nightlies url in test_logs.html
sh '''#!/bin/bash -e sh '''#!/bin/bash -e
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
@ -602,7 +602,7 @@ pipeline {
] ]
) )
]) ])
// remove the big test logs zip file, store the nightlies url in test_logs.txt // remove the big test logs zip file, store the nightlies url in test_logs.html
sh '''#!/bin/bash -e sh '''#!/bin/bash -e
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
@ -708,7 +708,7 @@ pipeline {
else else
echo "No archiver directory, skipping compressing." echo "No archiver directory, skipping compressing."
fi fi
''' '''
sshPublisher(publishers: [ sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies', sshPublisherDesc(configName: 'Nightlies',
transfers: [ transfers: [
@ -718,7 +718,7 @@ pipeline {
] ]
) )
]) ])
// remove the big test logs zip file, store the nightlies url in test_logs.txt // remove the big test logs zip file, store the nightlies url in test_logs.html
sh '''#!/bin/bash -e sh '''#!/bin/bash -e
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
@ -727,7 +727,7 @@ pipeline {
else else
echo "No test_logs.zip, skipping" echo "No test_logs.zip, skipping"
fi fi
''' '''
// Has to be relative to WORKSPACE. // Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
@ -770,14 +770,14 @@ pipeline {
rm -rf ".m2-for-src" && mkdir ".m2-for-src" rm -rf ".m2-for-src" && mkdir ".m2-for-src"
echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile
echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile
''' '''
sh '''#!/usr/bin/env bash sh '''#!/usr/bin/env bash
set -e set -e
rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
echo "got the following saved stats in 'output-srctarball/machine'" echo "got the following saved stats in 'output-srctarball/machine'"
ls -lh "output-srctarball/machine" ls -lh "output-srctarball/machine"
''' '''
sh """#!/bin/bash -e sh """#!/bin/bash -e
echo "Checking the steps for an RM to make a source artifact, then a binary artifact." echo "Checking the steps for an RM to make a source artifact, then a binary artifact."
if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \ if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
@ -792,7 +792,7 @@ pipeline {
echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
exit 1 exit 1
fi fi
""" """
echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'" echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'"
sh '''#!/bin/bash -e sh '''#!/bin/bash -e
if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then
@ -803,7 +803,7 @@ pipeline {
tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install" tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install"
client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | tail -n 1) client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | tail -n 1)
tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client" tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client"
''' '''
unstash 'hadoop-2' unstash 'hadoop-2'
sh '''#!/bin/bash -xe sh '''#!/bin/bash -xe
if [[ "${BRANCH}" = branch-2* ]] || [[ "${BRANCH}" = branch-1* ]]; then if [[ "${BRANCH}" = branch-2* ]] || [[ "${BRANCH}" = branch-1* ]]; then
@ -827,7 +827,7 @@ pipeline {
else else
echo "Skipping to run against Hadoop 2 for branch ${BRANCH}" echo "Skipping to run against Hadoop 2 for branch ${BRANCH}"
fi fi
''' '''
unstash 'hadoop-3' unstash 'hadoop-3'
sh '''#!/bin/bash -e sh '''#!/bin/bash -e
if [[ "${BRANCH}" = branch-1* ]]; then if [[ "${BRANCH}" = branch-1* ]]; then
@ -868,13 +868,31 @@ pipeline {
fi fi
echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
fi fi
''' '''
} }
post { post {
always { always {
stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile" stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile"
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
transfers: [
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
sourceFiles: "output-srctarball/hbase-src.tar.gz"
)
]
)
])
// remove the big src tarball, store the nightlies url in hbase-src.html
sh '''#!/bin/bash -e
SRC_TAR="${WORKSPACE}/output-srctarball/hbase-src.tar.gz"
if [ -f "${SRC_TAR}" ]; then
echo "Remove ${SRC_TAR} for saving space"
rm -rf "${SRC_TAR}"
python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
else
echo "No hbase-src.tar.gz, skipping"
fi
'''
archiveArtifacts artifacts: 'output-srctarball/*' archiveArtifacts artifacts: 'output-srctarball/*'
archiveArtifacts artifacts: 'output-srctarball/**/*' archiveArtifacts artifacts: 'output-srctarball/**/*'
archiveArtifacts artifacts: 'output-integration/*' archiveArtifacts artifacts: 'output-integration/*'