From 90ac3c93c9113cfe9451ffc48ec0f6cc397e9457 Mon Sep 17 00:00:00 2001 From: Apekshit Sharma Date: Wed, 20 Dec 2017 13:14:35 -0800 Subject: [PATCH] HBASE-19491 Improvements to Nighly runs: Fixes findbugs tests, Exclude flaky tests from master. --- dev-support/Jenkinsfile | 1 + dev-support/hbase-personality.sh | 117 +++++++++++++++++++---------- dev-support/hbase_nightly_yetus.sh | 8 ++ 3 files changed, 88 insertions(+), 38 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 83c0afb758a..fda329992b1 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -51,6 +51,7 @@ pipeline { // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure. TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite' BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile" + // Only used for master branch. EXCLUDE_TESTS_URL = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/' } parameters { diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index f33faea5a20..cf5eacded6b 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -64,7 +64,25 @@ function personality_globals # Override the maven options MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}" +} +## @description Parse extra arguments required by personalities, if any. +## @audience private +## @stability evolving +function personality_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --exclude-tests-url=*) + EXCLUDE_TESTS_URL=${i#*=} + ;; + --include-tests-url=*) + INCLUDE_TESTS_URL=${i#*=} + ;; + esac + done } ## @description Queue up modules for this personality @@ -77,19 +95,35 @@ function personality_modules local repostatus=$1 local testtype=$2 local extra="" + local MODULES=(${CHANGED_MODULES[@]}) - yetus_debug "Personality: ${repostatus} ${testtype}" + yetus_info "Personality: ${repostatus} ${testtype}" clear_personality_queue extra="-DHBasePatchProcess" - if [[ ${testtype} == mvninstall ]] || [[ "${BUILDMODE}" == full ]]; then + # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on + # full source code instead. In this case, do full compiles, tests, etc instead of per + # module. + # Used in nightly runs. + # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual + # modules if root is included. HBASE-18505 + if [[ "${BUILDMODE}" == "full" ]] || \ + [[ ( "${testtype}" == unit || "${testtype}" == compile ) && "${MODULES[*]}" =~ \. ]]; then + MODULES=(.) + fi + + if [[ ${testtype} == mvninstall ]]; then personality_enqueue_module . ${extra} return fi - if [[ ${testtype} = findbugs ]]; then + if [[ ${testtype} == findbugs ]]; then + # Run findbugs on each module individually to diff pre-patch and post-patch results and + # report new warnings for changed modules only. + # For some reason, findbugs on root is not working, but running on individual modules is + # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491 for module in "${CHANGED_MODULES[@]}"; do # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing # in src/main/java where findbugs goes to look @@ -108,40 +142,10 @@ function personality_modules # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the # tests respectively. - if [[ ${testtype} = unit ]]; then - # if the modules include root, skip all the submodules HBASE-18505 - if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then - CHANGED_MODULES=(.) - fi - - extra="${extra} -PrunAllTests" - yetus_debug "EXCLUDE_TESTS_URL = ${EXCLUDE_TESTS_URL}" - yetus_debug "INCLUDE_TESTS_URL = ${INCLUDE_TESTS_URL}" - if [[ -n "$EXCLUDE_TESTS_URL" ]]; then - if wget "$EXCLUDE_TESTS_URL" -O "excludes"; then - excludes=$(cat excludes) - yetus_debug "excludes=${excludes}" - if [[ -n "${excludes}" ]]; then - extra="${extra} -Dtest.exclude.pattern=${excludes}" - fi - rm excludes - else - echo "Wget error $? in fetching excludes file from url" \ - "${EXCLUDE_TESTS_URL}. Ignoring and proceeding." - fi - elif [[ -n "$INCLUDE_TESTS_URL" ]]; then - if wget "$INCLUDE_TESTS_URL" -O "includes"; then - includes=$(cat includes) - yetus_debug "includes=${includes}" - if [[ -n "${includes}" ]]; then - extra="${extra} -Dtest=${includes}" - fi - rm includes - else - echo "Wget error $? in fetching includes file from url" \ - "${INCLUDE_TESTS_URL}. Ignoring and proceeding." - fi - fi + if [[ ${testtype} == unit ]]; then + local tests_arg="" + get_include_exclude_tests_arg tests_arg + extra="${extra} -PrunAllTests ${tests_arg}" # Inject the jenkins build-id for our surefire invocations # Used by zombie detection stuff, even though we're not including that yet. @@ -150,12 +154,49 @@ function personality_modules fi fi - for module in "${CHANGED_MODULES[@]}"; do + for module in "${MODULES[@]}"; do # shellcheck disable=SC2086 personality_enqueue_module ${module} ${extra} done } +## @description Uses relevant include/exclude env variable to fetch list of included/excluded +# tests and sets given variable to arguments to be passes to maven command. +## @audience private +## @stability evolving +## @param name of variable to set with maven arguments +function get_include_exclude_tests_arg +{ + local __resultvar=$1 + yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}" + yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}" + if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then + if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then + excludes=$(cat excludes) + yetus_debug "excludes=${excludes}" + if [[ -n "${excludes}" ]]; then + eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'" + fi + rm excludes + else + yetus_error "Wget error $? in fetching excludes file from url" \ + "${EXCLUDE_TESTS_URL}. Ignoring and proceeding." + fi + elif [[ -n "$INCLUDE_TESTS_URL" ]]; then + if wget "$INCLUDE_TESTS_URL" -O "includes"; then + includes=$(cat includes) + yetus_debug "includes=${includes}" + if [[ -n "${includes}" ]]; then + eval "${__resultvar}='-Dtest=${includes}'" + fi + rm includes + else + yetus_error "Wget error $? in fetching includes file from url" \ + "${INCLUDE_TESTS_URL}. Ignoring and proceeding." + fi + fi +} + ################################################### # Below here are our one-off tests specific to hbase. # TODO break them into individual files so it's easier to maintain them? diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index 007d64addbf..da94c32582e 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -67,6 +67,14 @@ YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}") YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}") YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}") +# Currently, flaky list is calculated only for master branch. +if [[ -n "${EXCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then + YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}") +fi +if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then + YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}") +fi + if [[ true == "${DEBUG}" ]]; then YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}") fi