Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26186 jenkins script for caching artifacts should verify cached file before relying on it. (#3590) 

Signed-off-by: Michael Stack <stack@apache.org>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
Sean Busbey 2021-10-09 00:19:43 -05:00 committed by Duo Zhang
parent ef514cc35e
commit 1d8570cde1
2 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dev-support/Jenkinsfile vendored
View File

@ -91,6 +91,7 @@ pipeline {
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--working-dir "${WORKSPACE}/downloads-yetus" \
--keys 'https://www.apache.org/dist/yetus/KEYS' \
--verify-tar-gz \
"${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
"yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
@ -137,6 +138,7 @@ pipeline {
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--working-dir "${WORKSPACE}/downloads-hadoop-2" \
--keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
--verify-tar-gz \
"${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \
"hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz"
for stale in $(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | grep -v ${HADOOP2_VERSION}); do
@ -164,6 +166,7 @@ pipeline {
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--working-dir "${WORKSPACE}/downloads-hadoop-3" \
--keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
--verify-tar-gz \
"${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \
"hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz"
for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do

18
dev-support/jenkins-scripts/cache-apache-project-artifact.sh
View File

@ -21,6 +21,7 @@ function usage {
echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
echo ""
echo " --force for a redownload even if /path/to/download/file.tar.gz exists."
echo " --verify-tar-gz Only use a cached file if it can be parsed as a gzipped tarball."
echo " --working-dir /path/to/use Path for writing tempfiles. must exist."
echo " defaults to making a directory via mktemp that we clean."
echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download."
@ -35,6 +36,7 @@ fi
# Get arguments
declare done_if_cached="true"
declare verify_tar_gz="false"
declare working_dir
declare cleanup="true"
declare keys
@ -42,6 +44,7 @@ while [ $# -gt 0 ]
do
case "$1" in
--force) shift; done_if_cached="false";;
--verify-tar-gz) shift; verify_tar_gz="true";;
--working-dir) shift; working_dir=$1; cleanup="false"; shift;;
--keys) shift; keys=$1; shift;;
--) shift; break;;
@ -58,9 +61,18 @@ fi
target="$1"
artifact="$2"
if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then
echo "Reusing existing download of '${artifact}'."
exit 0
if [ -f "${target}" ] && [ -s "${target}" ] && [ -r "${target}" ] && [ "true" = "${done_if_cached}" ]; then
if [ "false" = "${verify_tar_gz}" ]; then
echo "Reusing existing download of '${artifact}'."
exit 0
fi
if ! tar tzf "${target}" > /dev/null 2>&1; then
echo "Cached artifact is not a well formed gzipped tarball; clearing the cached file at '${target}'."
rm -rf "${target}"
else
echo "Reusing existing download of '${artifact}', which is a well formed gzipped tarball."
exit 0
fi
fi
if [ -z "${working_dir}" ]; then