From 1d8570cde1fcc3840547a8571601fe0e2ed2fcfb Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Sat, 9 Oct 2021 00:19:43 -0500 Subject: [PATCH] HBASE-26186 jenkins script for caching artifacts should verify cached file before relying on it. (#3590) Signed-off-by: Michael Stack Signed-off-by: Duo Zhang --- dev-support/Jenkinsfile | 3 +++ .../cache-apache-project-artifact.sh | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index c588acff33a..983938d7f5d 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -91,6 +91,7 @@ pipeline { "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ --working-dir "${WORKSPACE}/downloads-yetus" \ --keys 'https://www.apache.org/dist/yetus/KEYS' \ + --verify-tar-gz \ "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \ "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz" mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz @@ -137,6 +138,7 @@ pipeline { "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ --working-dir "${WORKSPACE}/downloads-hadoop-2" \ --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \ + --verify-tar-gz \ "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \ "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz" for stale in $(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | grep -v ${HADOOP2_VERSION}); do @@ -164,6 +166,7 @@ pipeline { "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ --working-dir "${WORKSPACE}/downloads-hadoop-3" \ --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \ + --verify-tar-gz \ "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do diff --git a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh index 5653b05cb4e..ddd65b69a0f 100755 --- a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh +++ b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh @@ -21,6 +21,7 @@ function usage { echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz" echo "" echo " --force for a redownload even if /path/to/download/file.tar.gz exists." + echo " --verify-tar-gz Only use a cached file if it can be parsed as a gzipped tarball." echo " --working-dir /path/to/use Path for writing tempfiles. must exist." echo " defaults to making a directory via mktemp that we clean." echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download." @@ -35,6 +36,7 @@ fi # Get arguments declare done_if_cached="true" +declare verify_tar_gz="false" declare working_dir declare cleanup="true" declare keys @@ -42,6 +44,7 @@ while [ $# -gt 0 ] do case "$1" in --force) shift; done_if_cached="false";; + --verify-tar-gz) shift; verify_tar_gz="true";; --working-dir) shift; working_dir=$1; cleanup="false"; shift;; --keys) shift; keys=$1; shift;; --) shift; break;; @@ -58,9 +61,18 @@ fi target="$1" artifact="$2" -if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then - echo "Reusing existing download of '${artifact}'." - exit 0 +if [ -f "${target}" ] && [ -s "${target}" ] && [ -r "${target}" ] && [ "true" = "${done_if_cached}" ]; then + if [ "false" = "${verify_tar_gz}" ]; then + echo "Reusing existing download of '${artifact}'." + exit 0 + fi + if ! tar tzf "${target}" > /dev/null 2>&1; then + echo "Cached artifact is not a well formed gzipped tarball; clearing the cached file at '${target}'." + rm -rf "${target}" + else + echo "Reusing existing download of '${artifact}', which is a well formed gzipped tarball." + exit 0 + fi fi if [ -z "${working_dir}" ]; then