HBASE-19491 Improvements to Nighly runs: Fixes findbugs tests, Exclude flaky tests from master.
This commit is contained in:
Apekshit Sharma
parent
3cbe2c331b
commit
90ac3c93c9
|
|
@ -51,6 +51,7 @@ pipeline {
|
|||
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
|
||||
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
|
||||
BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
|
||||
// Only used for master branch.
|
||||
EXCLUDE_TESTS_URL = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
|
||||
}
|
||||
parameters {
|
||||
|
|
|
|||
|
|
@ -64,7 +64,25 @@ function personality_globals
|
|||
|
||||
# Override the maven options
|
||||
MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
|
||||
}
|
||||
|
||||
## @description Parse extra arguments required by personalities, if any.
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
function personality_parse_args
|
||||
{
|
||||
declare i
|
||||
|
||||
for i in "$@"; do
|
||||
case ${i} in
|
||||
--exclude-tests-url=*)
|
||||
EXCLUDE_TESTS_URL=${i#*=}
|
||||
;;
|
||||
--include-tests-url=*)
|
||||
INCLUDE_TESTS_URL=${i#*=}
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
## @description Queue up modules for this personality
|
||||
|
|
@ -77,19 +95,35 @@ function personality_modules
|
|||
local repostatus=$1
|
||||
local testtype=$2
|
||||
local extra=""
|
||||
local MODULES=(${CHANGED_MODULES[@]})
|
||||
|
||||
yetus_debug "Personality: ${repostatus} ${testtype}"
|
||||
yetus_info "Personality: ${repostatus} ${testtype}"
|
||||
|
||||
clear_personality_queue
|
||||
|
||||
extra="-DHBasePatchProcess"
|
||||
|
||||
if [[ ${testtype} == mvninstall ]] || [[ "${BUILDMODE}" == full ]]; then
|
||||
# BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
|
||||
# full source code instead. In this case, do full compiles, tests, etc instead of per
|
||||
# module.
|
||||
# Used in nightly runs.
|
||||
# If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
|
||||
# modules if root is included. HBASE-18505
|
||||
if [[ "${BUILDMODE}" == "full" ]] || \
|
||||
[[ ( "${testtype}" == unit || "${testtype}" == compile ) && "${MODULES[*]}" =~ \. ]]; then
|
||||
MODULES=(.)
|
||||
fi
|
||||
|
||||
if [[ ${testtype} == mvninstall ]]; then
|
||||
personality_enqueue_module . ${extra}
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ ${testtype} = findbugs ]]; then
|
||||
if [[ ${testtype} == findbugs ]]; then
|
||||
# Run findbugs on each module individually to diff pre-patch and post-patch results and
|
||||
# report new warnings for changed modules only.
|
||||
# For some reason, findbugs on root is not working, but running on individual modules is
|
||||
# working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
|
||||
for module in "${CHANGED_MODULES[@]}"; do
|
||||
# skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
|
||||
# in src/main/java where findbugs goes to look
|
||||
|
|
@ -108,40 +142,10 @@ function personality_modules
|
|||
# If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
|
||||
# and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
|
||||
# tests respectively.
|
||||
if [[ ${testtype} = unit ]]; then
|
||||
# if the modules include root, skip all the submodules HBASE-18505
|
||||
if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
|
||||
CHANGED_MODULES=(.)
|
||||
fi
|
||||
|
||||
extra="${extra} -PrunAllTests"
|
||||
yetus_debug "EXCLUDE_TESTS_URL = ${EXCLUDE_TESTS_URL}"
|
||||
yetus_debug "INCLUDE_TESTS_URL = ${INCLUDE_TESTS_URL}"
|
||||
if [[ -n "$EXCLUDE_TESTS_URL" ]]; then
|
||||
if wget "$EXCLUDE_TESTS_URL" -O "excludes"; then
|
||||
excludes=$(cat excludes)
|
||||
yetus_debug "excludes=${excludes}"
|
||||
if [[ -n "${excludes}" ]]; then
|
||||
extra="${extra} -Dtest.exclude.pattern=${excludes}"
|
||||
fi
|
||||
rm excludes
|
||||
else
|
||||
echo "Wget error $? in fetching excludes file from url" \
|
||||
"${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
|
||||
fi
|
||||
elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
|
||||
if wget "$INCLUDE_TESTS_URL" -O "includes"; then
|
||||
includes=$(cat includes)
|
||||
yetus_debug "includes=${includes}"
|
||||
if [[ -n "${includes}" ]]; then
|
||||
extra="${extra} -Dtest=${includes}"
|
||||
fi
|
||||
rm includes
|
||||
else
|
||||
echo "Wget error $? in fetching includes file from url" \
|
||||
"${INCLUDE_TESTS_URL}. Ignoring and proceeding."
|
||||
fi
|
||||
fi
|
||||
if [[ ${testtype} == unit ]]; then
|
||||
local tests_arg=""
|
||||
get_include_exclude_tests_arg tests_arg
|
||||
extra="${extra} -PrunAllTests ${tests_arg}"
|
||||
|
||||
# Inject the jenkins build-id for our surefire invocations
|
||||
# Used by zombie detection stuff, even though we're not including that yet.
|
||||
|
|
@ -150,12 +154,49 @@ function personality_modules
|
|||
fi
|
||||
fi
|
||||
|
||||
for module in "${CHANGED_MODULES[@]}"; do
|
||||
for module in "${MODULES[@]}"; do
|
||||
# shellcheck disable=SC2086
|
||||
personality_enqueue_module ${module} ${extra}
|
||||
done
|
||||
}
|
||||
|
||||
## @description Uses relevant include/exclude env variable to fetch list of included/excluded
|
||||
# tests and sets given variable to arguments to be passes to maven command.
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @param name of variable to set with maven arguments
|
||||
function get_include_exclude_tests_arg
|
||||
{
|
||||
local __resultvar=$1
|
||||
yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
|
||||
yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
|
||||
if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
|
||||
if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
|
||||
excludes=$(cat excludes)
|
||||
yetus_debug "excludes=${excludes}"
|
||||
if [[ -n "${excludes}" ]]; then
|
||||
eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
|
||||
fi
|
||||
rm excludes
|
||||
else
|
||||
yetus_error "Wget error $? in fetching excludes file from url" \
|
||||
"${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
|
||||
fi
|
||||
elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
|
||||
if wget "$INCLUDE_TESTS_URL" -O "includes"; then
|
||||
includes=$(cat includes)
|
||||
yetus_debug "includes=${includes}"
|
||||
if [[ -n "${includes}" ]]; then
|
||||
eval "${__resultvar}='-Dtest=${includes}'"
|
||||
fi
|
||||
rm includes
|
||||
else
|
||||
yetus_error "Wget error $? in fetching includes file from url" \
|
||||
"${INCLUDE_TESTS_URL}. Ignoring and proceeding."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
###################################################
|
||||
# Below here are our one-off tests specific to hbase.
|
||||
# TODO break them into individual files so it's easier to maintain them?
|
||||
|
|
|
|||
|
|
@ -67,6 +67,14 @@ YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
|
|||
YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
|
||||
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
|
||||
|
||||
# Currently, flaky list is calculated only for master branch.
|
||||
if [[ -n "${EXCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
|
||||
YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
|
||||
fi
|
||||
if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
|
||||
YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
|
||||
fi
|
||||
|
||||
if [[ true == "${DEBUG}" ]]; then
|
||||
YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in New Issue