HBASE-19124 Move nightly test of source artifact creation from Jenkinsfile to a script.
Signed-off-by: Apekshit Sharma <appy@apache.org>
This commit is contained in:
Sean Busbey
parent
53ec8bd648
commit
0ff9dabe6c
|
|
@ -258,51 +258,26 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
|
||||||
jdk "JDK 1.8 (latest)"
|
jdk "JDK 1.8 (latest)"
|
||||||
}
|
}
|
||||||
steps {
|
steps {
|
||||||
sh 'mvn --version --offline'
|
|
||||||
dir ("component") {
|
|
||||||
sh '''#!/bin/bash -e
|
sh '''#!/bin/bash -e
|
||||||
echo "Clean..."
|
echo "Setting up directories"
|
||||||
mvn --batch-mode -DskipTests clean
|
rm -rf "output-srctarball" && mkdir "output-srctarball"
|
||||||
echo "Step 3 Build the source tarball"
|
rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
|
||||||
mvn -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
|
rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
|
||||||
|
rm -rf ".m2-for-src" && mkdir ".m2-for-src"
|
||||||
'''
|
'''
|
||||||
|
sh """#!/bin/bash -e
|
||||||
|
${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh \
|
||||||
|
--intermediate-file-dir output-srctarball \
|
||||||
|
--unpack-temp-dir unpacked_src_tarball \
|
||||||
|
--maven-m2-initial .m2-for-repo \
|
||||||
|
--maven-m2-src-build .m2-for-src \
|
||||||
|
${env.BASEDIR}
|
||||||
|
"""
|
||||||
}
|
}
|
||||||
dir ("unpacked_src_tarball") {
|
post {
|
||||||
sh '''#!/bin/bash -e
|
always {
|
||||||
echo "Unpack the source tarball"
|
archive 'output-srctarball/*'
|
||||||
tar --strip-components=1 -xzf ../component/hbase-assembly/target/hbase-*-src.tar.gz
|
|
||||||
echo "Building from source artifact."
|
|
||||||
mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
|
|
||||||
echo "Clean up after checking ability to build."
|
|
||||||
mvn -DskipTests --batch-mode clean
|
|
||||||
'''
|
|
||||||
}
|
}
|
||||||
dir ("component") {
|
|
||||||
sh '''#!/bin/bash -e
|
|
||||||
echo "Clean up the source checkout"
|
|
||||||
mvn -DskipTests --batch-mode clean
|
|
||||||
echo "Diff against source tree"
|
|
||||||
diff --binary --recursive . ../unpacked_src_tarball >../diff_output || true
|
|
||||||
'''
|
|
||||||
}
|
|
||||||
// expectation check largely based on HBASE-14952
|
|
||||||
sh '''#!/bin/bash -e
|
|
||||||
echo "Checking against things we don't expect to include in the source tarball (git related, hbase-native-client, etc.)"
|
|
||||||
cat >known_excluded <<END
|
|
||||||
Only in .: .git
|
|
||||||
Only in .: .gitattributes
|
|
||||||
Only in .: .gitignore
|
|
||||||
Only in .: hbase-native-client
|
|
||||||
END
|
|
||||||
if ! diff known_excluded diff_output >/dev/null ; then
|
|
||||||
echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch."
|
|
||||||
echo "The expected differences are on the < side and the current differences are on the > side."
|
|
||||||
echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball."
|
|
||||||
diff known_excluded diff_output
|
|
||||||
else
|
|
||||||
echo "Everything looks as expected."
|
|
||||||
fi
|
|
||||||
'''
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,168 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
function usage {
|
||||||
|
echo "Usage: ${0} [options] /path/to/component/checkout"
|
||||||
|
echo ""
|
||||||
|
echo " --intermediate-file-dir /path/to/use Path for writing listings and diffs. must exist."
|
||||||
|
echo " defaults to making a directory in /tmp."
|
||||||
|
echo " --unpack-temp-dir /path/to/use Path for unpacking tarball. default to"
|
||||||
|
echo " 'unpacked_src_tarball' in intermediate directory."
|
||||||
|
echo " --maven-m2-initial /path/to/use Path for maven artifacts while building in"
|
||||||
|
echo " component-dir."
|
||||||
|
echo " --maven-m2-src-build /path/to/use Path for maven artifacts while building from the"
|
||||||
|
echo " unpacked source tarball."
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
# if no args specified, show usage
|
||||||
|
if [ $# -lt 1 ]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get arguments
|
||||||
|
declare component_dir
|
||||||
|
declare unpack_dir
|
||||||
|
declare m2_initial
|
||||||
|
declare m2_tarbuild
|
||||||
|
declare working_dir
|
||||||
|
while [ $# -gt 0 ]
|
||||||
|
do
|
||||||
|
case "$1" in
|
||||||
|
--unpack-temp-dir) shift; unpack_dir=$1; shift;;
|
||||||
|
--maven-m2-initial) shift; m2_initial=$1; shift;;
|
||||||
|
--maven-m2-src-build) shift; m2_tarbuild=$1; shift;;
|
||||||
|
--intermediate-file-dir) shift; working_dir=$1; shift;;
|
||||||
|
--) shift; break;;
|
||||||
|
-*) usage ;;
|
||||||
|
*) break;; # terminate while loop
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# should still have where component checkout is.
|
||||||
|
if [ $# -lt 1 ]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
|
||||||
|
|
||||||
|
if [ -z "${working_dir}" ]; then
|
||||||
|
working_dir=/tmp
|
||||||
|
while [[ -e ${working_dir} ]]; do
|
||||||
|
working_dir=/tmp/hbase-srctarball-test-${RANDOM}.${RANDOM}
|
||||||
|
done
|
||||||
|
mkdir "${working_dir}"
|
||||||
|
else
|
||||||
|
# absolutes please
|
||||||
|
working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
|
||||||
|
if [ ! -d "${working_dir}" ]; then
|
||||||
|
echo "passed working directory '${working_dir}' must already exist."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "You'll find logs and temp files in ${working_dir}"
|
||||||
|
|
||||||
|
if [ -z "${unpack_dir}" ]; then
|
||||||
|
unpack_dir="${working_dir}/unpacked_src_tarball"
|
||||||
|
mkdir "${unpack_dir}"
|
||||||
|
else
|
||||||
|
# absolutes please
|
||||||
|
unpack_dir="$(cd "$(dirname "${unpack_dir}")"; pwd)/$(basename "${unpack_dir}")"
|
||||||
|
if [ ! -d "${unpack_dir}" ]; then
|
||||||
|
echo "passed directory for unpacking the source tarball '${unpack_dir}' must already exist."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
rm -rf "${unpack_dir:?}/*"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${m2_initial}" ]; then
|
||||||
|
m2_initial="${working_dir}/.m2-initial"
|
||||||
|
mkdir "${m2_initial}"
|
||||||
|
else
|
||||||
|
# absolutes please
|
||||||
|
m2_initial="$(cd "$(dirname "${m2_initial}")"; pwd)/$(basename "${m2_initial}")"
|
||||||
|
if [ ! -d "${m2_initial}" ]; then
|
||||||
|
echo "passed directory for storing the initial build's maven repo '${m2_initial}' " \
|
||||||
|
"must already exist."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${m2_tarbuild}" ]; then
|
||||||
|
m2_tarbuild="${working_dir}/.m2-tarbuild"
|
||||||
|
mkdir "${m2_tarbuild}"
|
||||||
|
else
|
||||||
|
# absolutes please
|
||||||
|
m2_tarbuild="$(cd "$(dirname "${m2_tarbuild}")"; pwd)/$(basename "${m2_tarbuild}")"
|
||||||
|
if [ ! -d "${m2_tarbuild}" ]; then
|
||||||
|
echo "passed directory for storing the build from src tarball's maven repo '${m2_tarbuild}' " \
|
||||||
|
"must already exist."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# This is meant to mimic what a release manager will do to create RCs.
|
||||||
|
# See http://hbase.apache.org/book.html#maven.release
|
||||||
|
|
||||||
|
echo "Maven details, in case our JDK doesn't match expectations:"
|
||||||
|
mvn --version --offline | tee "${working_dir}/maven_version"
|
||||||
|
|
||||||
|
echo "Do a clean building of the source artifact using code in ${component_dir}"
|
||||||
|
cd "${component_dir}"
|
||||||
|
echo "Clean..."
|
||||||
|
mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_before.log" 2>&1
|
||||||
|
echo "Step 3 Build the source tarball"
|
||||||
|
mvn -Prelease --batch-mode -Dmaven.repo.local="${m2_initial}" install -DskipTests assembly:single \
|
||||||
|
-Dassembly.file=hbase-assembly/src/main/assembly/src.xml \
|
||||||
|
>"${working_dir}/component_build_src_tarball.log" 2>&1
|
||||||
|
|
||||||
|
cd "${unpack_dir}"
|
||||||
|
echo "Unpack the source tarball"
|
||||||
|
tar --strip-components=1 -xzf "${component_dir}"/hbase-assembly/target/hbase-*-src.tar.gz \
|
||||||
|
>"${working_dir}/srctarball_unpack.log" 2>&1
|
||||||
|
echo "Building from source artifact."
|
||||||
|
mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
|
||||||
|
assembly:single >"${working_dir}/srctarball_install.log" 2>&1
|
||||||
|
|
||||||
|
echo "Clean up after checking ability to build."
|
||||||
|
mvn -DskipTests --batch-mode clean >"${working_dir}/srctarball_clean.log" 2>&1
|
||||||
|
|
||||||
|
|
||||||
|
cd "${component_dir}"
|
||||||
|
echo "Clean up the source checkout"
|
||||||
|
mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_after.log" 2>&1
|
||||||
|
echo "Diff against source tree"
|
||||||
|
diff --binary --recursive . "${unpack_dir}" >"${working_dir}/diff_output" || true
|
||||||
|
|
||||||
|
cd "${working_dir}"
|
||||||
|
# expectation check largely based on HBASE-14952
|
||||||
|
echo "Checking against things we don't expect to include in the source tarball (git related, hbase-native-client, etc.)"
|
||||||
|
cat >known_excluded <<END
|
||||||
|
Only in .: .git
|
||||||
|
Only in .: .gitattributes
|
||||||
|
Only in .: .gitignore
|
||||||
|
Only in .: hbase-native-client
|
||||||
|
END
|
||||||
|
if ! diff known_excluded diff_output >"${working_dir}/unexpected.diff" ; then
|
||||||
|
echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch."
|
||||||
|
echo "The expected differences are on the < side and the current differences are on the > side."
|
||||||
|
echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball."
|
||||||
|
diff known_excluded diff_output
|
||||||
|
else
|
||||||
|
echo "Everything looks as expected."
|
||||||
|
fi
|
||||||
Loading…
Reference in New Issue