From 0ff9dabe6c808af0f0cdfe3c328fef957ec815e7 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Wed, 1 Nov 2017 08:12:03 -0500 Subject: [PATCH] HBASE-19124 Move nightly test of source artifact creation from Jenkinsfile to a script. Signed-off-by: Apekshit Sharma --- dev-support/Jenkinsfile | 61 ++----- dev-support/hbase_nightly_source-artifact.sh | 168 +++++++++++++++++++ 2 files changed, 186 insertions(+), 43 deletions(-) create mode 100755 dev-support/hbase_nightly_source-artifact.sh diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index d7faeb60677..5bd41aa36ff 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -258,51 +258,26 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}" jdk "JDK 1.8 (latest)" } steps { - sh 'mvn --version --offline' - dir ("component") { - sh '''#!/bin/bash -e - echo "Clean..." - mvn --batch-mode -DskipTests clean - echo "Step 3 Build the source tarball" - mvn -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml -''' - } - dir ("unpacked_src_tarball") { - sh '''#!/bin/bash -e - echo "Unpack the source tarball" - tar --strip-components=1 -xzf ../component/hbase-assembly/target/hbase-*-src.tar.gz - echo "Building from source artifact." - mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single - echo "Clean up after checking ability to build." - mvn -DskipTests --batch-mode clean -''' - } - dir ("component") { - sh '''#!/bin/bash -e - echo "Clean up the source checkout" - mvn -DskipTests --batch-mode clean - echo "Diff against source tree" - diff --binary --recursive . ../unpacked_src_tarball >../diff_output || true -''' - } - // expectation check largely based on HBASE-14952 sh '''#!/bin/bash -e - echo "Checking against things we don't expect to include in the source tarball (git related, hbase-native-client, etc.)" - cat >known_excluded </dev/null ; then - echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch." - echo "The expected differences are on the < side and the current differences are on the > side." - echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball." - diff known_excluded diff_output - else - echo "Everything looks as expected." - fi + echo "Setting up directories" + rm -rf "output-srctarball" && mkdir "output-srctarball" + rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball" + rm -rf ".m2-for-repo" && mkdir ".m2-for-repo" + rm -rf ".m2-for-src" && mkdir ".m2-for-src" ''' + sh """#!/bin/bash -e + ${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh \ + --intermediate-file-dir output-srctarball \ + --unpack-temp-dir unpacked_src_tarball \ + --maven-m2-initial .m2-for-repo \ + --maven-m2-src-build .m2-for-src \ + ${env.BASEDIR} +""" + } + post { + always { + archive 'output-srctarball/*' + } } } } diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh new file mode 100755 index 00000000000..4e4c6a20de3 --- /dev/null +++ b/dev-support/hbase_nightly_source-artifact.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +function usage { + echo "Usage: ${0} [options] /path/to/component/checkout" + echo "" + echo " --intermediate-file-dir /path/to/use Path for writing listings and diffs. must exist." + echo " defaults to making a directory in /tmp." + echo " --unpack-temp-dir /path/to/use Path for unpacking tarball. default to" + echo " 'unpacked_src_tarball' in intermediate directory." + echo " --maven-m2-initial /path/to/use Path for maven artifacts while building in" + echo " component-dir." + echo " --maven-m2-src-build /path/to/use Path for maven artifacts while building from the" + echo " unpacked source tarball." + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare component_dir +declare unpack_dir +declare m2_initial +declare m2_tarbuild +declare working_dir +while [ $# -gt 0 ] +do + case "$1" in + --unpack-temp-dir) shift; unpack_dir=$1; shift;; + --maven-m2-initial) shift; m2_initial=$1; shift;; + --maven-m2-src-build) shift; m2_tarbuild=$1; shift;; + --intermediate-file-dir) shift; working_dir=$1; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have where component checkout is. +if [ $# -lt 1 ]; then + usage +fi +component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" + +if [ -z "${working_dir}" ]; then + working_dir=/tmp + while [[ -e ${working_dir} ]]; do + working_dir=/tmp/hbase-srctarball-test-${RANDOM}.${RANDOM} + done + mkdir "${working_dir}" +else + # absolutes please + working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")" + if [ ! -d "${working_dir}" ]; then + echo "passed working directory '${working_dir}' must already exist." + exit 1 + fi +fi + +echo "You'll find logs and temp files in ${working_dir}" + +if [ -z "${unpack_dir}" ]; then + unpack_dir="${working_dir}/unpacked_src_tarball" + mkdir "${unpack_dir}" +else + # absolutes please + unpack_dir="$(cd "$(dirname "${unpack_dir}")"; pwd)/$(basename "${unpack_dir}")" + if [ ! -d "${unpack_dir}" ]; then + echo "passed directory for unpacking the source tarball '${unpack_dir}' must already exist." + exit 1 + fi + rm -rf "${unpack_dir:?}/*" +fi + +if [ -z "${m2_initial}" ]; then + m2_initial="${working_dir}/.m2-initial" + mkdir "${m2_initial}" +else + # absolutes please + m2_initial="$(cd "$(dirname "${m2_initial}")"; pwd)/$(basename "${m2_initial}")" + if [ ! -d "${m2_initial}" ]; then + echo "passed directory for storing the initial build's maven repo '${m2_initial}' " \ + "must already exist." + exit 1 + fi +fi + +if [ -z "${m2_tarbuild}" ]; then + m2_tarbuild="${working_dir}/.m2-tarbuild" + mkdir "${m2_tarbuild}" +else + # absolutes please + m2_tarbuild="$(cd "$(dirname "${m2_tarbuild}")"; pwd)/$(basename "${m2_tarbuild}")" + if [ ! -d "${m2_tarbuild}" ]; then + echo "passed directory for storing the build from src tarball's maven repo '${m2_tarbuild}' " \ + "must already exist." + exit 1 + fi +fi + +# This is meant to mimic what a release manager will do to create RCs. +# See http://hbase.apache.org/book.html#maven.release + +echo "Maven details, in case our JDK doesn't match expectations:" +mvn --version --offline | tee "${working_dir}/maven_version" + +echo "Do a clean building of the source artifact using code in ${component_dir}" +cd "${component_dir}" +echo "Clean..." +mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_before.log" 2>&1 +echo "Step 3 Build the source tarball" +mvn -Prelease --batch-mode -Dmaven.repo.local="${m2_initial}" install -DskipTests assembly:single \ + -Dassembly.file=hbase-assembly/src/main/assembly/src.xml \ + >"${working_dir}/component_build_src_tarball.log" 2>&1 + +cd "${unpack_dir}" +echo "Unpack the source tarball" +tar --strip-components=1 -xzf "${component_dir}"/hbase-assembly/target/hbase-*-src.tar.gz \ + >"${working_dir}/srctarball_unpack.log" 2>&1 +echo "Building from source artifact." +mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \ + assembly:single >"${working_dir}/srctarball_install.log" 2>&1 + +echo "Clean up after checking ability to build." +mvn -DskipTests --batch-mode clean >"${working_dir}/srctarball_clean.log" 2>&1 + + +cd "${component_dir}" +echo "Clean up the source checkout" +mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_after.log" 2>&1 +echo "Diff against source tree" +diff --binary --recursive . "${unpack_dir}" >"${working_dir}/diff_output" || true + +cd "${working_dir}" +# expectation check largely based on HBASE-14952 +echo "Checking against things we don't expect to include in the source tarball (git related, hbase-native-client, etc.)" +cat >known_excluded <"${working_dir}/unexpected.diff" ; then + echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch." + echo "The expected differences are on the < side and the current differences are on the > side." + echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball." + diff known_excluded diff_output +else + echo "Everything looks as expected." +fi