#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### BUILD_URL is set by Hudson if it is run by patch process this="${BASH_SOURCE-$0}" BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) CWD=$(pwd) USER_PARAMS=("$@") GLOBALTIMER=$(date +"%s") ## @description Setup the default global variables ## @audience public ## @stability stable ## @replaceable no function setup_defaults { if [[ -z "${MAVEN_HOME:-}" ]]; then MVN=mvn else MVN=${MAVEN_HOME}/bin/mvn fi # This parameter needs to be kept as an array MAVEN_ARGS=() PROJECT_NAME=hadoop HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute" JENKINS=false BASEDIR=$(pwd) RELOCATE_PATCH_DIR=false USER_PLUGIN_DIR="" LOAD_SYSTEM_PLUGINS=true FINDBUGS_HOME=${FINDBUGS_HOME:-} FINDBUGS_WARNINGS_FAIL_PRECHECK=false ECLIPSE_HOME=${ECLIPSE_HOME:-} BUILD_NATIVE=${BUILD_NATIVE:-true} PATCH_BRANCH="" PATCH_BRANCH_DEFAULT="trunk" CHANGED_MODULES="" USER_MODULE_LIST="" OFFLINE=false CHANGED_FILES="" REEXECED=false RESETREPO=false ISSUE="" ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' TIMER=$(date +"%s") PATCHURL="" OSTYPE=$(uname -s) # Solaris needs POSIX, not SVID case ${OSTYPE} in SunOS) PS=${PS:-ps} AWK=${AWK:-/usr/xpg4/bin/awk} SED=${SED:-/usr/xpg4/bin/sed} WGET=${WGET:-wget} GIT=${GIT:-git} EGREP=${EGREP:-/usr/xpg4/bin/egrep} GREP=${GREP:-/usr/xpg4/bin/grep} PATCH=${PATCH:-patch} DIFF=${DIFF:-/usr/gnu/bin/diff} JIRACLI=${JIRA:-jira} FILE=${FILE:-file} ;; *) PS=${PS:-ps} AWK=${AWK:-awk} SED=${SED:-sed} WGET=${WGET:-wget} GIT=${GIT:-git} EGREP=${EGREP:-egrep} GREP=${GREP:-grep} PATCH=${PATCH:-patch} DIFF=${DIFF:-diff} JIRACLI=${JIRA:-jira} FILE=${FILE:-file} ;; esac declare -a JIRA_COMMENT_TABLE declare -a JIRA_FOOTER_TABLE declare -a JIRA_HEADER declare -a JIRA_TEST_TABLE JFC=0 JTC=0 JTT=0 RESULT=0 } ## @description Print a message to stderr ## @audience public ## @stability stable ## @replaceable no ## @param string function hadoop_error { echo "$*" 1>&2 } ## @description Print a message to stderr if --debug is turned on ## @audience public ## @stability stable ## @replaceable no ## @param string function hadoop_debug { if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then echo "[$(date) DEBUG]: $*" 1>&2 fi } ## @description Activate the local timer ## @audience public ## @stability stable ## @replaceable no function start_clock { hadoop_debug "Start clock" TIMER=$(date +"%s") } ## @description Print the elapsed time in seconds since the start of the local timer ## @audience public ## @stability stable ## @replaceable no function stop_clock { local -r stoptime=$(date +"%s") local -r elapsed=$((stoptime-TIMER)) hadoop_debug "Stop clock" echo ${elapsed} } ## @description Print the elapsed time in seconds since the start of the global timer ## @audience private ## @stability stable ## @replaceable no function stop_global_clock { local -r stoptime=$(date +"%s") local -r elapsed=$((stoptime-GLOBALTIMER)) hadoop_debug "Stop global clock" echo ${elapsed} } ## @description Add time to the local timer ## @audience public ## @stability stable ## @replaceable no ## @param seconds function offset_clock { ((TIMER=TIMER-$1)) } ## @description Add to the header of the display ## @audience public ## @stability stable ## @replaceable no ## @param string function add_jira_header { JIRA_HEADER[${JHC}]="| $* |" JHC=$(( JHC+1 )) } ## @description Add to the output table. If the first parameter is a number ## @description that is the vote for that column and calculates the elapsed time ## @description based upon the last start_clock(). If it the string null, then it is ## @description a special entry that signifies extra ## @description content for the final column. The second parameter is the reporting ## @description subsystem (or test) that is providing the vote. The second parameter ## @description is always required. The third parameter is any extra verbage that goes ## @description with that subsystem. ## @audience public ## @stability stable ## @replaceable no ## @param +1/0/-1/null ## @param subsystem ## @param string ## @return Elapsed time display function add_jira_table { local value=$1 local subsystem=$2 shift 2 local color local calctime=0 local -r elapsed=$(stop_clock) if [[ ${elapsed} -lt 0 ]]; then calctime="N/A" else printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) fi echo "" echo "Elapsed time: ${calctime}" echo "" case ${value} in 1|+1) value="+1" color="green" ;; -1) color="red" ;; 0) color="blue" ;; null) ;; esac if [[ -z ${color} ]]; then JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |" JTC=$(( JTC+1 )) else JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |" JTC=$(( JTC+1 )) fi } ## @description Put the final environment information at the bottom ## @description of the footer table ## @stability stable ## @audience private ## @replaceable yes function close_jira_footer { # shellcheck disable=SC2016 local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \") local -r unamea=$(uname -a) add_jira_footer "Java" "${javaversion}" add_jira_footer "uname" "${unamea}" } ## @description Put the final elapsed time at the bottom of the table. ## @audience private ## @stability stable ## @replaceable no function close_jira_table { local -r elapsed=$(stop_global_clock) if [[ ${elapsed} -lt 0 ]]; then calctime="N/A" else printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) fi echo "" echo "Total Elapsed time: ${calctime}" echo "" JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |" JTC=$(( JTC+1 )) } ## @description Add to the footer of the display. @@BASE@@ will get replaced with the ## @description correct location for the local filesystem in dev mode or the URL for ## @description Jenkins mode. ## @audience public ## @stability stable ## @replaceable no ## @param subsystem ## @param string function add_jira_footer { local subsystem=$1 shift 1 JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |" JFC=$(( JFC+1 )) } ## @description Special table just for unit test failures ## @audience public ## @stability stable ## @replaceable no ## @param failurereason ## @param testlist function add_jira_test_table { local failure=$1 shift 1 JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |" JTT=$(( JTT+1 )) } ## @description Large display for the user console ## @audience public ## @stability stable ## @replaceable no ## @param string ## @return large chunk of text function big_console_header { local text="$*" local spacing=$(( (75+${#text}) /2 )) printf "\n\n" echo "============================================================================" echo "============================================================================" printf "%*s\n" ${spacing} "${text}" echo "============================================================================" echo "============================================================================" printf "\n\n" } ## @description Remove {color} tags from a string ## @audience public ## @stability stable ## @replaceable no ## @param string ## @return string function colorstripper { local string=$1 shift 1 local green="" local white="" local red="" local blue="" echo "${string}" | \ ${SED} -e "s,{color:red},${red},g" \ -e "s,{color:green},${green},g" \ -e "s,{color:blue},${blue},g" \ -e "s,{color},${white},g" } ## @description Find the largest size of a column of an array ## @audience private ## @stability evolving ## @replaceable no ## @return size function findlargest { local column=$1 shift local a=("$@") local sizeofa=${#a[@]} local i=0 until [[ ${i} -gt ${sizeofa} ]]; do # shellcheck disable=SC2086 string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| ) if [[ ${#string} -gt $maxlen ]]; then maxlen=${#string} fi i=$((i+1)) done echo "${maxlen}" } ## @description Verify that ${JAVA_HOME} is defined ## @audience public ## @stability stable ## @replaceable no ## @return 1 - no JAVA_HOME ## @return 0 - JAVA_HOME defined function find_java_home { start_clock if [[ -z ${JAVA_HOME:-} ]]; then case $(uname -s) in Darwin) if [[ -z "${JAVA_HOME}" ]]; then if [[ -x /usr/libexec/java_home ]]; then JAVA_HOME="$(/usr/libexec/java_home)" export JAVA_HOME else export JAVA_HOME=/Library/Java/Home fi fi ;; *) ;; esac fi if [[ -z ${JAVA_HOME:-} ]]; then echo "JAVA_HOME is not defined." add_jira_table -1 pre-patch "JAVA_HOME is not defined." return 1 fi return 0 } ## @description Write the contents of a file to jenkins ## @params filename ## @stability stable ## @audience public ## @returns ${JIRACLI} exit code function write_to_jira { local -r commentfile=${1} shift local retval if [[ ${OFFLINE} == false && ${JENKINS} == true ]]; then export USER=hudson # shellcheck disable=SC2086 ${JIRACLI} --comment "$(cat ${commentfile})" \ -s https://issues.apache.org/jira \ -a addcomment -u hadoopqa \ -p "${JIRA_PASSWD}" \ --issue "${ISSUE}" retval=$? ${JIRACLI} -s https://issues.apache.org/jira \ -a logout -u hadoopqa \ -p "${JIRA_PASSWD}" fi return ${retval} } ## @description Verify that the patch directory is still in working order ## @description since bad actors on some systems wipe it out. If not, ## @description recreate it and then exit ## @audience private ## @stability evolving ## @replaceable yes ## @returns may exit on failure function verify_patchdir_still_exists { local -r commentfile=/tmp/testpatch.$$.${RANDOM} local extra="" if [[ ! -d ${PATCH_DIR} ]]; then rm "${commentfile}" 2>/dev/null echo "(!) The patch artifact directory has been removed! " > "${commentfile}" echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}" echo cat ${commentfile} echo if [[ ${JENKINS} == true ]]; then if [[ -n ${NODE_NAME} ]]; then extra=" (node ${NODE_NAME})" fi echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}" write_to_jira ${commentfile} fi rm "${commentfile}" cleanup_and_exit ${RESULT} fi } ## @description generate a list of all files and line numbers that ## @description that were added/changed in the source repo ## @audience private ## @stability stable ## @params filename ## @replaceable no function compute_gitdiff { local outfile=$1 local file local line local startline local counter local numlines local actual pushd "${BASEDIR}" >/dev/null while read line; do if [[ ${line} =~ ^\+\+\+ ]]; then file="./"$(echo "${line}" | cut -f2- -d/) continue elif [[ ${line} =~ ^@@ ]]; then startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + ) numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, ) # if this is empty, then just this line # if it is 0, then no lines were added and this part of the patch # is strictly a delete if [[ ${numlines} == 0 ]]; then continue elif [[ -z ${numlines} ]]; then numlines=1 fi counter=0 until [[ ${counter} -gt ${numlines} ]]; do ((actual=counter+startline)) echo "${file}:${actual}:" >> "${outfile}" ((counter=counter+1)) done fi done < <("${GIT}" diff --unified=0 --no-color) popd >/dev/null } ## @description Print the command to be executing to the screen. Then ## @description run the command, sending stdout and stderr to the given filename ## @description This will also ensure that any directories in ${BASEDIR} have ## @description the exec bit set as a pre-exec step. ## @audience public ## @stability stable ## @param filename ## @param command ## @param [..] ## @replaceable no ## @returns $? function echo_and_redirect { local logfile=$1 shift verify_patchdir_still_exists find "${BASEDIR}" -type d -exec chmod +x {} \; echo "${*} > ${logfile} 2>&1" "${@}" > "${logfile}" 2>&1 } ## @description is PATCH_DIR relative to BASEDIR? ## @audience public ## @stability stable ## @replaceable yes ## @returns 1 - no, PATCH_DIR ## @returns 0 - yes, PATCH_DIR - BASEDIR function relative_patchdir { local p=${PATCH_DIR#${BASEDIR}} if [[ ${#p} -eq ${#PATCH_DIR} ]]; then echo ${p} return 1 fi p=${p#/} echo ${p} return 0 } ## @description Print the usage information ## @audience public ## @stability stable ## @replaceable no function hadoop_usage { local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]') echo "Usage: test-patch.sh [options] patch-file | issue-number | http" echo echo "Where:" echo " patch-file is a local patch file containing the changes to test" echo " issue-number is a 'Patch Available' JIRA defect number (e.g. '${up}-9902') to test" echo " http is an HTTP address to download the patch file" echo echo "Options:" echo "--basedir= The directory to apply the patch to (default current directory)" echo "--branch= Forcibly set the branch" echo "--branch-default= If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')" echo "--build-native= If true, then build native components (default 'true')" echo "--contrib-guide= URL to point new users towards project conventions. (default Hadoop's wiki)" echo "--debug If set, then output some extra stuff to stderr" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail" echo "--issue-re= Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')" echo "--modulelist= Specify additional modules to test (comma delimited)" echo "--offline Avoid connecting to the Internet" echo "--patch-dir= The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')" echo "--plugins= A directory of user provided plugins. see test-patch.d for examples (default empty)" echo "--project= The short name for project currently using test-patch (default 'hadoop')" echo "--resetrepo Forcibly clean the repo" echo "--run-tests Run all relevant tests below the base directory" echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d" echo "--testlist= Specify which subsystem tests to use (comma delimited)" echo "Shell binary overrides:" echo "--awk-cmd= The 'awk' command to use (default 'awk')" echo "--diff-cmd= The GNU-compatible 'diff' command to use (default 'diff')" echo "--file-cmd= The 'file' command to use (default 'file')" echo "--git-cmd= The 'git' command to use (default 'git')" echo "--grep-cmd= The 'grep' command to use (default 'grep')" echo "--mvn-cmd= The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" echo "--patch-cmd= The 'patch' command to use (default 'patch')" echo "--ps-cmd= The 'ps' command to use (default 'ps')" echo "--sed-cmd= The 'sed' command to use (default 'sed')" echo echo "Jenkins-only options:" echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)" echo "--jira-cmd= The 'jira' command to use (default 'jira')" echo "--jira-password= The password for the 'jira' command" echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup." echo "--wget-cmd= The 'wget' command to use (default 'wget')" } ## @description Interpret the command line parameters ## @audience private ## @stability stable ## @replaceable no ## @params $@ ## @return May exit on failure function parse_args { local i local j for i in "$@"; do case ${i} in --awk-cmd=*) AWK=${i#*=} ;; --basedir=*) BASEDIR=${i#*=} ;; --branch=*) PATCH_BRANCH=${i#*=} ;; --branch-default=*) PATCH_BRANCH_DEFAULT=${i#*=} ;; --build-native=*) BUILD_NATIVE=${i#*=} ;; --contrib-guide=*) HOW_TO_CONTRIBUTE=${i#*=} ;; --debug) HADOOP_SHELL_SCRIPT_DEBUG=true ;; --diff-cmd=*) DIFF=${i#*=} ;; --dirty-workspace) DIRTY_WORKSPACE=true ;; --eclipse-home=*) ECLIPSE_HOME=${i#*=} ;; --file-cmd=*) FILE=${i#*=} ;; --findbugs-home=*) FINDBUGS_HOME=${i#*=} ;; --findbugs-strict-precheck) FINDBUGS_WARNINGS_FAIL_PRECHECK=true ;; --git-cmd=*) GIT=${i#*=} ;; --grep-cmd=*) GREP=${i#*=} ;; --help|-help|-h|help|--h|--\?|-\?|\?) hadoop_usage exit 0 ;; --issue-re=*) ISSUE_RE=${i#*=} ;; --java-home=*) JAVA_HOME=${i#*=} ;; --jenkins) JENKINS=true ;; --jira-cmd=*) JIRACLI=${i#*=} ;; --jira-password=*) JIRA_PASSWD=${i#*=} ;; --modulelist=*) USER_MODULE_LIST=${i#*=} USER_MODULE_LIST=${USER_MODULE_LIST//,/ } hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}" ;; --mvn-cmd=*) MVN=${i#*=} ;; --mv-patch-dir) RELOCATE_PATCH_DIR=true; ;; --offline) OFFLINE=true ;; --patch-cmd=*) PATCH=${i#*=} ;; --patch-dir=*) USER_PATCH_DIR=${i#*=} ;; --plugins=*) USER_PLUGIN_DIR=${i#*=} ;; --project=*) PROJECT_NAME=${i#*=} ;; --ps-cmd=*) PS=${i#*=} ;; --reexec) REEXECED=true start_clock add_jira_table 0 reexec "dev-support patch detected." ;; --resetrepo) RESETREPO=true ;; --run-tests) RUN_TESTS=true ;; --skip-system-plugins) LOAD_SYSTEM_PLUGINS=false ;; --testlist=*) testlist=${i#*=} testlist=${testlist//,/ } for j in ${testlist}; do hadoop_debug "Manually adding patch test subsystem ${j}" add_test "${j}" done ;; --wget-cmd=*) WGET=${i#*=} ;; *) PATCH_OR_ISSUE=${i} ;; esac done # if we requested offline, pass that to mvn if [[ ${OFFLINE} == "true" ]] ; then MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline) fi # we need absolute dir for ${BASEDIR} cd "${CWD}" BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P) if [[ ${BUILD_NATIVE} == "true" ]] ; then NATIVE_PROFILE=-Pnative REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop fi if [[ -z "${PATCH_OR_ISSUE}" ]]; then hadoop_usage exit 1 fi if [[ ${JENKINS} == "true" ]] ; then echo "Running in Jenkins mode" ISSUE=${PATCH_OR_ISSUE} RESETREPO=true # shellcheck disable=SC2034 ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}" else if [[ ${RESETREPO} == "true" ]] ; then echo "Running in destructive (--resetrepo) developer mode" else echo "Running in developer mode" fi JENKINS=false fi if [[ -n ${USER_PATCH_DIR} ]]; then PATCH_DIR="${USER_PATCH_DIR}" else PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$ fi cd "${CWD}" if [[ ! -d ${PATCH_DIR} ]]; then mkdir -p "${PATCH_DIR}" if [[ $? == 0 ]] ; then echo "${PATCH_DIR} has been created" else echo "Unable to create ${PATCH_DIR}" cleanup_and_exit 1 fi fi # we need absolute dir for PATCH_DIR PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt } ## @description Locate the pom.xml file for a given directory ## @audience private ## @stability stable ## @replaceable no ## @return directory containing the pom.xml function find_pom_dir { local dir dir=$(dirname "$1") hadoop_debug "Find pom dir for: ${dir}" while builtin true; do if [[ -f "${dir}/pom.xml" ]];then echo "${dir}" hadoop_debug "Found: ${dir}" return else dir=$(dirname "${dir}") fi done } ## @description List of files that ${PATCH_DIR}/patch modifies ## @audience private ## @stability stable ## @replaceable no ## @return None; sets ${CHANGED_FILES} function find_changed_files { # get a list of all of the files that have been changed, # except for /dev/null (which would be present for new files). # Additionally, remove any a/ b/ patterns at the front # of the patch filenames and any revision info at the end # shellcheck disable=SC2016 CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \ | ${SED} \ -e 's,^....,,' \ -e 's,^[ab]/,,' \ | ${GREP} -v /dev/null \ | ${AWK} '{print $1}' \ | sort -u) } ## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies ## @audience private ## @stability stable ## @replaceable no ## @return None; sets ${CHANGED_MODULES} function find_changed_modules { # Come up with a list of changed files into ${TMP} local pomdirs local module local pommods # Now find all the modules that were changed for file in ${CHANGED_FILES}; do #shellcheck disable=SC2086 pomdirs="${pomdirs} $(find_pom_dir ${file})" done # Filter out modules without code for module in ${pomdirs}; do ${GREP} "pom" "${module}/pom.xml" > /dev/null if [[ "$?" != 0 ]]; then pommods="${pommods} ${module}" fi done #shellcheck disable=SC2086 CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u) } ## @description git checkout the appropriate branch to test. Additionally, this calls ## @description 'determine_issue' and 'determine_branch' based upon the context provided ## @description in ${PATCH_DIR} and in git after checkout. ## @audience private ## @stability stable ## @replaceable no ## @return 0 on success. May exit on failure. function git_checkout { local currentbranch local exemptdir big_console_header "Confirming git environment" cd "${BASEDIR}" if [[ ! -d .git ]]; then hadoop_error "ERROR: ${BASEDIR} is not a git repo." cleanup_and_exit 1 fi if [[ ${RESETREPO} == "true" ]] ; then ${GIT} reset --hard if [[ $? != 0 ]]; then hadoop_error "ERROR: git reset is failing" cleanup_and_exit 1 fi # if PATCH_DIR is in BASEDIR, then we don't want # git wiping it out. exemptdir=$(relative_patchdir) if [[ $? == 1 ]]; then ${GIT} clean -xdf else # we do, however, want it emptied of all _files_. # we need to leave _directories_ in case we are in # re-exec mode (which places a directory full of stuff in it) hadoop_debug "Exempting ${exemptdir} from clean" rm "${PATCH_DIR}/*" 2>/dev/null ${GIT} clean -xdf -e "${exemptdir}" fi if [[ $? != 0 ]]; then hadoop_error "ERROR: git clean is failing" cleanup_and_exit 1 fi ${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}" if [[ $? != 0 ]]; then hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing" cleanup_and_exit 1 fi determine_branch if [[ ${PATCH_BRANCH} =~ ^git ]]; then PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2) fi # we need to explicitly fetch in case the # git ref hasn't been brought in tree yet if [[ ${OFFLINE} == false ]]; then if [[ -f .git/rebase-apply ]]; then hadoop_error "ERROR: previous rebase failed. Aborting it." ${GIT} rebase --abort fi ${GIT} pull --rebase if [[ $? != 0 ]]; then hadoop_error "ERROR: git pull is failing" cleanup_and_exit 1 fi fi # forcibly checkout this branch or git ref ${GIT} checkout --force "${PATCH_BRANCH}" if [[ $? != 0 ]]; then hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing" cleanup_and_exit 1 fi # if we've selected a feature branch that has new changes # since our last build, we'll need to rebase to see those changes. if [[ ${OFFLINE} == false ]]; then ${GIT} pull --rebase if [[ $? != 0 ]]; then hadoop_error "ERROR: git pull is failing" cleanup_and_exit 1 fi fi else status=$(${GIT} status --porcelain) if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then hadoop_error "ERROR: --dirty-workspace option not provided." hadoop_error "ERROR: can't run in a workspace that contains the following modifications" hadoop_error "${status}" cleanup_and_exit 1 fi determine_branch if [[ ${PATCH_BRANCH} =~ ^git ]]; then PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2) fi currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD) if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." echo "WARNING: Continuing anyway..." PATCH_BRANCH=${currentbranch} fi fi determine_issue GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD) # shellcheck disable=SC2034 VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum} if [[ "${ISSUE}" == 'Unknown' ]]; then echo "Testing patch on ${PATCH_BRANCH}." else echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." fi add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}" if [[ ! -f ${BASEDIR}/pom.xml ]]; then hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting." add_jira_table -1 pre-patch "Unsupported build system." output_to_jira 1 cleanup_and_exit 1 fi return 0 } ## @description Confirm the source environment is compilable ## @audience private ## @stability stable ## @replaceable no ## @return 0 on success ## @return 1 on failure function precheck_without_patch { local -r mypwd=$(pwd) big_console_header "Pre-patch ${PATCH_BRANCH} Java verification" start_clock verify_needed_test javac if [[ $? == 1 ]]; then echo "Compiling ${mypwd}" echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch if [[ $? != 0 ]] ; then echo "${PATCH_BRANCH} compilation is broken?" add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken." return 1 fi else echo "Patch does not appear to need javac tests." fi verify_needed_test javadoc if [[ $? == 1 ]]; then echo "Javadoc'ing ${mypwd}" echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess if [[ $? != 0 ]] ; then echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?" add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken." return 1 fi else echo "Patch does not appear to need javadoc tests." fi verify_needed_test site if [[ $? == 1 ]]; then echo "site creation for ${mypwd}" echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess if [[ $? != 0 ]] ; then echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?" add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken." return 1 fi else echo "Patch does not appear to need site tests." fi precheck_findbugs if [[ $? != 0 ]] ; then return 1 fi add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy." return 0 } ## @description Confirm the given branch is a member of the list of space ## @description delimited branches or a git ref ## @audience private ## @stability evolving ## @replaceable no ## @param branch ## @param branchlist ## @return 0 on success ## @return 1 on failure function verify_valid_branch { local branches=$1 local check=$2 local i # shortcut some common # non-resolvable names if [[ -z ${check} ]]; then return 1 fi if [[ ${check} == patch ]]; then return 1 fi if [[ ${check} =~ ^git ]]; then ref=$(echo "${check}" | cut -f2 -dt) count=$(echo "${ref}" | wc -c | tr -d ' ') if [[ ${count} == 8 || ${count} == 41 ]]; then return 0 fi return 1 fi for i in ${branches}; do if [[ "${i}" == "${check}" ]]; then return 0 fi done return 1 } ## @description Try to guess the branch being tested using a variety of heuristics ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success, with PATCH_BRANCH updated appropriately ## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT function determine_branch { local allbranches local patchnamechunk hadoop_debug "Determine branch" # something has already set this, so move on if [[ -n ${PATCH_BRANCH} ]]; then return fi pushd "${BASEDIR}" > /dev/null # developer mode, existing checkout, whatever if [[ "${DIRTY_WORKSPACE}" == true ]];then PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD) echo "dirty workspace mode; applying against existing branch" return fi allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g) for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do hadoop_debug "Determine branch: starting with ${j}" # shellcheck disable=SC2016 patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}') # ISSUE.branch.##.patch hadoop_debug "Determine branch: ISSUE.branch.##.patch" PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. ) verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" if [[ $? == 0 ]]; then return fi # ISSUE-branch-##.patch hadoop_debug "Determine branch: ISSUE-branch-##.patch" PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-) verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" if [[ $? == 0 ]]; then return fi # ISSUE-##.patch.branch hadoop_debug "Determine branch: ISSUE-##.patch.branch" # shellcheck disable=SC2016 PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}') verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" if [[ $? == 0 ]]; then return fi # ISSUE-branch.##.patch hadoop_debug "Determine branch: ISSUE-branch.##.patch" # shellcheck disable=SC2016 PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null) verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" if [[ $? == 0 ]]; then return fi done PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}" popd >/dev/null } ## @description Try to guess the issue being tested using a variety of heuristics ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success, with ISSUE updated appropriately ## @return 1 on failure, with ISSUE updated to "Unknown" function determine_issue { local patchnamechunk local maybeissue hadoop_debug "Determine issue" # we can shortcut jenkins if [[ ${JENKINS} == true ]]; then ISSUE=${PATCH_OR_ISSUE} return 0 fi # shellcheck disable=SC2016 patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}') maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-) if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then ISSUE=${maybeissue} return 0 fi ISSUE="Unknown" return 1 } ## @description Add the given test type ## @audience public ## @stability stable ## @replaceable yes ## @param test function add_test { local testname=$1 hadoop_debug "Testing against ${testname}" if [[ -z ${NEEDED_TESTS} ]]; then hadoop_debug "Setting tests to ${testname}" NEEDED_TESTS=${testname} elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then hadoop_debug "Adding ${testname}" NEEDED_TESTS="${NEEDED_TESTS} ${testname}" fi } ## @description Verify if a given test was requested ## @audience public ## @stability stable ## @replaceable yes ## @param test ## @return 1 = yes ## @return 0 = no function verify_needed_test { local i=$1 if [[ ${NEEDED_TESTS} =~ $i ]]; then return 1 fi return 0 } ## @description Use some heuristics to determine which long running ## @description tests to run ## @audience private ## @stability stable ## @replaceable no function determine_needed_tests { local i for i in ${CHANGED_FILES}; do if [[ ${i} =~ src/main/webapp ]]; then hadoop_debug "tests/webapp: ${i}" elif [[ ${i} =~ \.sh || ${i} =~ \.cmd ]]; then hadoop_debug "tests/shell: ${i}" elif [[ ${i} =~ \.md$ || ${i} =~ \.md\.vm$ || ${i} =~ src/site || ${i} =~ src/main/docs ]]; then hadoop_debug "tests/site: ${i}" add_test site elif [[ ${i} =~ \.c$ || ${i} =~ \.cc$ || ${i} =~ \.h$ || ${i} =~ \.hh$ || ${i} =~ \.proto$ || ${i} =~ src/test || ${i} =~ \.cmake$ || ${i} =~ CMakeLists.txt ]]; then hadoop_debug "tests/units: ${i}" add_test javac add_test unit elif [[ ${i} =~ pom.xml$ || ${i} =~ \.java$ || ${i} =~ src/main ]]; then hadoop_debug "tests/javadoc+units: ${i}" add_test javadoc add_test javac add_test unit fi if [[ ${i} =~ \.java$ ]]; then add_test findbugs fi for plugin in ${PLUGINS}; do if declare -f ${plugin}_filefilter >/dev/null 2>&1; then "${plugin}_filefilter" "${i}" fi done done add_jira_footer "Optional Tests" "${NEEDED_TESTS}" } ## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the ## @description necessary work to place it into ${PATCH_DIR}/patch. ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure, may exit function locate_patch { local notSureIfPatch=false hadoop_debug "locate patch" if [[ -f ${PATCH_OR_ISSUE} ]]; then PATCH_FILE="${PATCH_OR_ISSUE}" else if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then echo "Patch is being downloaded at $(date) from" PATCHURL="${PATCH_OR_ISSUE}" else ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}" if [[ $? != 0 ]];then hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference." cleanup_and_exit 1 fi if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then if [[ ${JENKINS} == true ]]; then hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"." cleanup_and_exit 1 else hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"." fi fi relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*') PATCHURL="http://issues.apache.org${relativePatchURL}" if [[ ! ${PATCHURL} =~ \.patch$ ]]; then notSureIfPatch=true fi patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*') echo "${ISSUE} patch is being downloaded at $(date) from" fi echo "${PATCHURL}" add_jira_footer "Patch URL" "${PATCHURL}" ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}" if [[ $? != 0 ]];then hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded." cleanup_and_exit 1 fi PATCH_FILE="${PATCH_DIR}/patch" fi if [[ ! -f "${PATCH_DIR}/patch" ]]; then cp "${PATCH_FILE}" "${PATCH_DIR}/patch" if [[ $? == 0 ]] ; then echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}" else hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}" cleanup_and_exit 1 fi fi if [[ ${notSureIfPatch} == "true" ]]; then guess_patch_file "${PATCH_DIR}/patch" if [[ $? != 0 ]]; then hadoop_error "ERROR: ${PATCHURL} is not a patch file." cleanup_and_exit 1 else hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry." add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions." fi fi } ## @description Given a possible patch file, guess if it's a patch file without using smart-apply-patch ## @audience private ## @stability evolving ## @param path to patch file to test ## @return 0 we think it's a patch file ## @return 1 we think it's not a patch file function guess_patch_file { local patch=$1 local fileOutput hadoop_debug "Trying to guess is ${patch} is a patch file." fileOutput=$("${FILE}" "${patch}") if [[ $fileOutput =~ \ diff\ ]]; then hadoop_debug "file magic says it's a diff." return 0 fi fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$") if [[ $? == 0 ]]; then hadoop_debug "first line looks like a patch file." return 0 fi return 1 } ## @description Given ${PATCH_DIR}/patch, verify the patch is good using ${BINDIR}/smart-apply-patch.sh ## @description in dryrun mode. ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function verify_patch_file { # Before building, check to make sure that the patch is valid export PATCH "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun if [[ $? != 0 ]] ; then echo "PATCH APPLICATION FAILED" add_jira_table -1 patch "The patch command could not apply the patch during dryrun." return 1 else return 0 fi } ## @description Given ${PATCH_DIR}/patch, apply the patch using ${BINDIR}/smart-apply-patch.sh ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return exit on failure function apply_patch_file { big_console_header "Applying patch" export PATCH "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" if [[ $? != 0 ]] ; then echo "PATCH APPLICATION FAILED" ((RESULT = RESULT + 1)) add_jira_table -1 patch "The patch command could not apply the patch." output_to_console 1 output_to_jira 1 cleanup_and_exit 1 fi return 0 } ## @description If this actually patches the files used for the QA process ## @description under dev-support and its subdirectories, then ## @description run with the patched version for the test. ## @audience private ## @stability evolving ## @replaceable no ## @return none; otherwise relaunches function check_reexec { local commentfile=${PATCH_DIR}/tp.${RANDOM} if [[ ${REEXECED} == true ]]; then big_console_header "Re-exec mode detected. Continuing." return fi if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch && ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then return fi big_console_header "dev-support patch detected" if [[ ${RESETREPO} == false ]]; then ((RESULT = RESULT + 1)) hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :(" add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory." return fi printf "\n\nRe-executing against patched versions to test.\n\n" apply_patch_file if [[ ${JENKINS} == true ]]; then rm "${commentfile}" 2>/dev/null echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}" echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}" echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}" write_to_jira "${commentfile}" rm "${commentfile}" fi cd "${CWD}" mkdir -p "${PATCH_DIR}/dev-support-test" (cd "${BINDIR}"; tar cpf - . ) \ | (cd "${PATCH_DIR}/dev-support-test"; tar xpf - ) big_console_header "exec'ing test-patch.sh now..." exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \ --reexec \ --branch="${PATCH_BRANCH}" \ --patch-dir="${PATCH_DIR}" \ "${USER_PARAMS[@]}" } ## @description Check the current directory for @author tags ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_author { local authorTags big_console_header "Checking there are no @author tags in the patch." start_clock if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then add_jira_table 0 @author "Skipping @author checks as test-patch has been patched." return 0 fi authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch") echo "There appear to be ${authorTags} @author tags in the patch." if [[ ${authorTags} != 0 ]] ; then add_jira_table -1 @author \ "The patch appears to contain ${authorTags} @author tags which the Hadoop" \ " community has agreed to not allow in code contributions." return 1 fi add_jira_table +1 @author "The patch does not contain any @author tags." return 0 } ## @description Check the patch file for changed/new tests ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_modified_unittests { local testReferences=0 local i verify_needed_test unit if [[ $? == 0 ]]; then return 0 fi big_console_header "Checking there are new or changed tests in the patch." start_clock for i in ${CHANGED_FILES}; do if [[ ${i} =~ /test/ ]]; then ((testReferences=testReferences + 1)) fi done echo "There appear to be ${testReferences} test file(s) referenced in the patch." if [[ ${testReferences} == 0 ]] ; then add_jira_table -1 "tests included" \ "The patch doesn't appear to include any new or modified tests. " \ "Please justify why no new tests are needed for this patch." \ "Also please list what manual steps were performed to verify this patch." return 1 fi add_jira_table +1 "tests included" \ "The patch appears to include ${testReferences} new or modified test files." return 0 } ## @description Helper for check_javadoc ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function count_javadoc_warns { local warningfile=$1 #shellcheck disable=SC2016,SC2046 return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}') } ## @description Count and compare the number of JavaDoc warnings pre- and post- patch ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_javadoc { local numBranchJavadocWarnings local numPatchJavadocWarnings verify_needed_test javadoc if [[ $? == 0 ]]; then echo "This patch does not appear to need javadoc checks." return 0 fi big_console_header "Determining number of patched javadoc warnings" start_clock if [[ -d hadoop-project ]]; then (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1) fi if [[ -d hadoop-common-project/hadoop-annotations ]]; then (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1) fi echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" numBranchJavadocWarnings=$? count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt" numPatchJavadocWarnings=$? echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch." if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \ "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \ > "${PATCH_DIR}/diffJavadocWarnings.txt" rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" add_jira_table -1 javadoc "The applied patch generated "\ "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \ " additional warning messages." add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt" return 1 fi fi add_jira_table +1 javadoc "There were no new javadoc warning messages." return 0 } ## @description Make sure site still compiles ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_site { local -r mypwd=$(pwd) verify_needed_test site if [[ $? == 0 ]]; then echo "This patch does not appear to need site checks." return 0 fi big_console_header "Determining if patched site still builds" start_clock echo "site creation for ${mypwd}" echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess if [[ $? != 0 ]] ; then echo "Site compilation is broken" add_jira_table -1 site "Site compilation is broken." add_jira_footer site "@@BASE@@/patchSiteWarnings.txt" return 1 fi add_jira_table +1 site "Site still builds." return 0 } ## @description Helper for check_javac ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function count_javac_warns { local warningfile=$1 #shellcheck disable=SC2016,SC2046 return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}") } ## @description Count and compare the number of javac warnings pre- and post- patch ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_javac { local branchJavacWarnings local patchJavacWarnings verify_needed_test javac if [[ $? == 0 ]]; then echo "This patch does not appear to need javac checks." return 0 fi big_console_header "Determining number of patched javac warnings." start_clock echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch if [[ $? != 0 ]] ; then add_jira_table -1 javac "The patch appears to cause the build to fail." return 2 fi ### Compare ${PATCH_BRANCH} and patch javac warning numbers if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt" count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" branchJavacWarnings=$? count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt" patchJavacWarnings=$? echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch." if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \ "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \ > "${PATCH_DIR}/diffJavacWarnings.txt" add_jira_table -1 javac "The applied patch generated "\ "$((patchJavacWarnings-branchJavacWarnings))" \ " additional warning messages." add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt" return 1 fi fi fi add_jira_table +1 javac "There were no new javac warning messages." return 0 } ## @description Verify all files have an Apache License ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_apachelicense { big_console_header "Determining number of patched release audit warnings." start_clock echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess #shellcheck disable=SC2038 find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt" ### Compare ${PATCH_BRANCH} and patch release audit warning numbers if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt") echo "" echo "" echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch." if [[ ${patchReleaseAuditWarnings} != "" ]] ; then if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings." ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \ > "${PATCH_DIR}/patchReleaseAuditProblems.txt" echo "Lines that start with ????? in the release audit "\ "report indicate files that do not have an Apache license header." \ >> "${PATCH_DIR}/patchReleaseAuditProblems.txt" add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt" return 1 fi fi fi add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings." return 0 } ## @description Verify mvn install works ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_mvn_install { local retval verify_needed_test javadoc retval=$? verify_needed_test javac ((retval = retval + $? )) if [[ ${retval} == 0 ]]; then echo "This patch does not appear to need mvn install checks." return 0 fi big_console_header "Installing all of the jars" start_clock echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess retval=$? if [[ ${retval} != 0 ]]; then add_jira_table -1 install "The patch causes mvn install to fail." else add_jira_table +1 install "mvn install still works." fi return ${retval} } ## @description are the needed bits for findbugs present? ## @audience private ## @stability evolving ## @replaceable no ## @return 0 findbugs will work for our use ## @return 1 findbugs is missing some component function findbugs_is_installed { if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs" add_jira_table -1 findbugs "Findbugs is not installed." return 1 fi return 0 } ## @description Run the maven findbugs plugin and record found issues in a bug database ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function findbugs_mvnrunner { local name=$1 local logfile=$2 local warnings_file=$3 echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \ "-D${PROJECT_NAME}PatchProcess" < /dev/null if [[ $? != 0 ]]; then return 1 fi cp target/findbugsXml.xml "${warnings_file}.xml" "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \ "${warnings_file}.xml" "${warnings_file}.xml" if [[ $? != 0 ]]; then return 1 fi "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \ "${warnings_file}.html" if [[ $? != 0 ]]; then return 1 fi return 0 } ## @description Track pre-existing findbugs warnings ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function precheck_findbugs { local -r mypwd=$(pwd) local module_suffix local modules=${CHANGED_MODULES} local module local findbugs_version local rc=0 local module_findbugs_warnings local findbugs_warnings=0 verify_needed_test findbugs if [[ $? == 0 ]]; then echo "Patch does not appear to need findbugs tests." return 0 fi echo "findbugs baseline for ${mypwd}" findbugs_is_installed if [[ $? != 0 ]]; then return 1 fi for module in ${modules} do pushd "${module}" >/dev/null echo " Running findbugs in ${module}" module_suffix=$(basename "${module}") findbugs_mvnrunner "${PATCH_BRANCH}" \ "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}" (( rc = rc + $? )) if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then #shellcheck disable=SC2016 module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ "${PATCH_BRANCH}" \ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \ | ${AWK} '{print $1}') if [[ $? != 0 ]]; then popd >/dev/null return 1 fi findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings)) if [[ ${module_findbugs_warnings} -gt 0 ]] ; then add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html" fi fi popd >/dev/null done #shellcheck disable=SC2016 findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt") if [[ ${rc} -ne 0 ]]; then echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?" add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}." return 1 fi if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \ ${findbugs_warnings} -gt 0 ]] ; then echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings." add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings." return 1 fi return 0 } ## @description Verify patch does not trigger any findbugs warnings ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_findbugs { local rc=0 local module local modules=${CHANGED_MODULES} local module_suffix local combined_xml local newBugs local new_findbugs_warnings local new_findbugs_fixed_warnings local findbugs_warnings=0 local findbugs_fixed_warnings=0 local line local firstpart local secondpart local findbugs_version verify_needed_test findbugs if [[ $? == 0 ]]; then return 0 fi big_console_header "Determining number of patched Findbugs warnings." start_clock findbugs_is_installed if [[ $? != 0 ]]; then return 1 fi for module in ${modules} do pushd "${module}" >/dev/null echo " Running findbugs in ${module}" module_suffix=$(basename "${module}") findbugs_mvnrunner patch \ "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \ "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}" if [[ $? != 0 ]] ; then ((rc = rc +1)) echo "Post-patch findbugs compilation is broken." add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken." continue fi combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml" newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}" "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \ -output "${combined_xml}" \ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \ "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" if [[ $? != 0 ]]; then popd >/dev/null return 1 fi #shellcheck disable=SC2016 new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \ "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}') if [[ $? != 0 ]]; then popd >/dev/null return 1 fi #shellcheck disable=SC2016 new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \ "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}') if [[ $? != 0 ]]; then popd >/dev/null return 1 fi echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings." findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings)) findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings)) "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \ "${newBugs}.html" if [[ $? != 0 ]]; then popd >/dev/null return 1 fi if [[ ${new_findbugs_warnings} -gt 0 ]] ; then populate_test_table FindBugs "module:${module_suffix}" while read line; do firstpart=$(echo "${line}" | cut -f2 -d:) secondpart=$(echo "${line}" | cut -f9- -d' ') add_jira_test_table "" "${firstpart}:${secondpart}" done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml") add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html" fi popd >/dev/null done #shellcheck disable=SC2016 findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt") if [[ ${findbugs_warnings} -gt 0 ]] ; then add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings." return 1 fi if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings." else add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." fi return 0 } ## @description Make sure Maven's eclipse generation works. ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_mvn_eclipse { big_console_header "Running mvn eclipse:eclipse." verify_needed_test javac if [[ $? == 0 ]]; then echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse" return 0 fi start_clock echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess if [[ $? != 0 ]] ; then add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse." return 1 fi add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse." return 0 } ## @description Utility to push many tests into the failure list ## @audience private ## @stability evolving ## @replaceable no ## @param testdesc ## @param testlist function populate_test_table { local reason=$1 shift local first="" local i for i in "$@"; do if [[ -z "${first}" ]]; then add_jira_test_table "${reason}" "${i}" first="${reason}" else add_jira_test_table " " "${i}" fi done } ## @description Run and verify the output of the appropriate unit tests ## @audience private ## @stability evolving ## @replaceable no ## @return 0 on success ## @return 1 on failure function check_unittests { verify_needed_test unit if [[ $? == 0 ]]; then echo "Existing unit tests do not test patched files. Skipping." return 0 fi big_console_header "Running unit tests" start_clock local failed_tests="" local modules=${CHANGED_MODULES} local building_common=0 local hdfs_modules local ordered_modules="" local failed_test_builds="" local test_timeouts="" local test_logfile local test_build_result local module_test_timeouts="" local result local totalresult=0 local module_prefix # # If we are building hadoop-hdfs-project, we must build the native component # of hadoop-common-project first. In order to accomplish this, we move the # hadoop-hdfs subprojects to the end of the list so that common will come # first. # # Of course, we may not be building hadoop-common at all-- in this case, we # explicitly insert a mvn compile -Pnative of common, to ensure that the # native libraries show up where we need them. # for module in ${modules}; do if [[ ${module} == hadoop-hdfs-project* ]]; then hdfs_modules="${hdfs_modules} ${module}" elif [[ ${module} == hadoop-common-project* ]]; then ordered_modules="${ordered_modules} ${module}" building_common=1 else ordered_modules="${ordered_modules} ${module}" fi done if [[ -n "${hdfs_modules}" ]]; then ordered_modules="${ordered_modules} ${hdfs_modules}" if [[ ${building_common} -eq 0 ]]; then echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests." echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess" if [[ $? != 0 ]]; then add_jira_table -1 "native" "Failed to build the native portion " \ "of hadoop-common prior to running the unit tests in ${ordered_modules}" return 1 else add_jira_table +1 "native" "Pre-build of native portion" fi fi fi for module in ${ordered_modules}; do result=0 start_clock pushd "${module}" >/dev/null module_suffix=$(basename "${module}") module_prefix=$(echo "${module}" | cut -f2 -d- ) test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt echo " Running tests in ${module_suffix}" echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} -D${PROJECT_NAME}PatchProcess test_build_result=$? add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt" # shellcheck disable=2016 module_test_timeouts=$(${AWK} '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' "${test_logfile}") if [[ -n "${module_test_timeouts}" ]] ; then test_timeouts="${test_timeouts} ${module_test_timeouts}" result=1 fi #shellcheck disable=SC2026,SC2038,SC2016 module_failed_tests=$(find . -name 'TEST*.xml'\ | xargs "${GREP}" -l -E "/dev/null if [[ $result == 1 ]]; then add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}." else add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}." fi ((totalresult = totalresult + result)) done if [[ -n "${failed_tests}" ]] ; then # shellcheck disable=SC2086 populate_test_table "Failed unit tests" ${failed_tests} fi if [[ -n "${test_timeouts}" ]] ; then # shellcheck disable=SC2086 populate_test_table "Timed out tests" ${test_timeouts} fi if [[ -n "${failed_test_builds}" ]] ; then # shellcheck disable=SC2086 populate_test_table "Failed build" ${failed_test_builds} fi if [[ ${JENKINS} == true ]]; then add_jira_footer "Test Results" "${BUILD_URL}testReport/" fi if [[ ${totalresult} -gt 0 ]]; then return 1 else return 0 fi } ## @description Print out the finished details on the console ## @audience private ## @stability evolving ## @replaceable no ## @param runresult ## @return 0 on success ## @return 1 on failure function output_to_console { local result=$1 shift local i local ourstring local vote local subs local ela local comment local commentfile1="${PATCH_DIR}/comment.1" local commentfile2="${PATCH_DIR}/comment.2" local normaltop local line local seccoladj=0 local spcfx=${PATCH_DIR}/spcl.txt if [[ ${result} == 0 ]]; then if [[ ${JENKINS} == false ]]; then { printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; printf "ICB8bXwgICB8bXwgIAo" } > "${spcfx}" fi printf "\n\n+1 overall\n\n" else if [[ ${JENKINS} == false ]]; then { printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf"; printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf"; printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98"; printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg"; printf "ICAgICAgICAgICAgICAgICAK" } > "${spcfx}" fi printf "\n\n-1 overall\n\n" fi if [[ -f ${spcfx} ]]; then if which base64 >/dev/null 2>&1; then base64 --decode "${spcfx}" 2>/dev/null elif which openssl >/dev/null 2>&1; then openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null fi echo echo rm "${spcfx}" fi seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}") if [[ ${seccoladj} -lt 10 ]]; then seccoladj=10 fi seccoladj=$((seccoladj + 2 )) i=0 until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do printf "%s\n" "${JIRA_HEADER[${i}]}" ((i=i+1)) done printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment" echo "============================================================================" i=0 until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ') vote=$(echo "${ourstring}" | cut -f2 -d\|) vote=$(colorstripper "${vote}") subs=$(echo "${ourstring}" | cut -f3 -d\|) ela=$(echo "${ourstring}" | cut -f4 -d\|) comment=$(echo "${ourstring}" | cut -f5 -d\|) echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}" normaltop=$(head -1 "${commentfile1}") ${SED} -e '1d' "${commentfile1}" > "${commentfile2}" printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \ "${subs}" "${ela}" "${normaltop}" while read line; do printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}" done < "${commentfile2}" ((i=i+1)) rm "${commentfile2}" "${commentfile1}" 2>/dev/null done if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}") printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason" i=0 until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ') vote=$(echo "${ourstring}" | cut -f2 -d\|) subs=$(echo "${ourstring}" | cut -f3 -d\|) printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}" ((i=i+1)) done fi printf "\n\n|| Subsystem || Report/Notes ||\n" echo "============================================================================" i=0 until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" | ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") printf "%s\n" "${comment}" ((i=i+1)) done } ## @description Print out the finished details to the JIRA issue ## @audience private ## @stability evolving ## @replaceable no ## @param runresult function output_to_jira { local result=$1 local i local commentfile=${PATCH_DIR}/commentfile local comment rm "${commentfile}" 2>/dev/null if [[ ${JENKINS} != "true" ]] ; then return 0 fi big_console_header "Adding comment to JIRA" add_jira_footer "Console output" "${BUILD_URL}console" if [[ ${result} == 0 ]]; then add_jira_header "(/) *{color:green}+1 overall{color}*" else add_jira_header "(x) *{color:red}-1 overall{color}*" fi { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" i=0 until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}" ((i=i+1)) done { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}" i=0 until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}" ((i=i+1)) done if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" echo "|| Reason || Tests ||" >> "${commentfile}" i=0 until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}" ((i=i+1)) done fi { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" echo "|| Subsystem || Report/Notes ||" >> "${commentfile}" i=0 until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" | ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g") printf "%s\n" "${comment}" >> "${commentfile}" ((i=i+1)) done printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" write_to_jira "${commentfile}" } ## @description Clean the filesystem as appropriate and then exit ## @audience private ## @stability evolving ## @replaceable no ## @param runresult function cleanup_and_exit { local result=$1 if [[ ${JENKINS} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \ -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then # if PATCH_DIR is already inside BASEDIR, then # there is no need to move it since we assume that # Jenkins or whatever already knows where it is at # since it told us to put it there! relative_patchdir >/dev/null if [[ $? == 1 ]]; then hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}" mv "${PATCH_DIR}" "${BASEDIR}" fi fi big_console_header "Finished build." # shellcheck disable=SC2086 exit ${result} } ## @description Driver to execute _postcheckout routines ## @audience private ## @stability evolving ## @replaceable no function postcheckout { local routine local plugin for routine in find_java_home verify_patch_file do verify_patchdir_still_exists hadoop_debug "Running ${routine}" ${routine} (( RESULT = RESULT + $? )) if [[ ${RESULT} != 0 ]] ; then output_to_console 1 output_to_jira 1 cleanup_and_exit 1 fi done for plugin in ${PLUGINS}; do verify_patchdir_still_exists if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then hadoop_debug "Running ${plugin}_postcheckout" #shellcheck disable=SC2086 ${plugin}_postcheckout (( RESULT = RESULT + $? )) if [[ ${RESULT} != 0 ]] ; then output_to_console 1 output_to_jira 1 cleanup_and_exit 1 fi fi done } ## @description Driver to execute _preapply routines ## @audience private ## @stability evolving ## @replaceable no function preapply { local routine local plugin for routine in precheck_without_patch check_author \ check_modified_unittests do verify_patchdir_still_exists hadoop_debug "Running ${routine}" ${routine} (( RESULT = RESULT + $? )) done for plugin in ${PLUGINS}; do verify_patchdir_still_exists if declare -f ${plugin}_preapply >/dev/null 2>&1; then hadoop_debug "Running ${plugin}_preapply" #shellcheck disable=SC2086 ${plugin}_preapply (( RESULT = RESULT + $? )) fi done } ## @description Driver to execute _postapply routines ## @audience private ## @stability evolving ## @replaceable no function postapply { local routine local plugin local retval compute_gitdiff "${GITDIFFLINES}" check_javac retval=$? if [[ ${retval} -gt 1 ]] ; then output_to_console 1 output_to_jira 1 cleanup_and_exit 1 fi ((RESULT = RESULT + retval)) for routine in check_javadoc check_apachelicense check_site do verify_patchdir_still_exists hadoop_debug "Running ${routine}" $routine (( RESULT = RESULT + $? )) done for plugin in ${PLUGINS}; do verify_patchdir_still_exists if declare -f ${plugin}_postapply >/dev/null 2>&1; then hadoop_debug "Running ${plugin}_postapply" #shellcheck disable=SC2086 ${plugin}_postapply (( RESULT = RESULT + $? )) fi done } ## @description Driver to execute _postinstall routines ## @audience private ## @stability evolving ## @replaceable no function postinstall { local routine local plugin for routine in check_mvn_eclipse check_findbugs do verify_patchdir_still_exists hadoop_debug "Running ${routine}" ${routine} (( RESULT = RESULT + $? )) done for plugin in ${PLUGINS}; do verify_patchdir_still_exists if declare -f ${plugin}_postinstall >/dev/null 2>&1; then hadoop_debug "Running ${plugin}_postinstall" #shellcheck disable=SC2086 ${plugin}_postinstall (( RESULT = RESULT + $? )) fi done } ## @description Driver to execute _tests routines ## @audience private ## @stability evolving ## @replaceable no function runtests { local plugin ### Run tests for Jenkins or if explictly asked for by a developer if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then verify_patchdir_still_exists check_unittests (( RESULT = RESULT + $? )) fi for plugin in ${PLUGINS}; do verify_patchdir_still_exists if declare -f ${plugin}_tests >/dev/null 2>&1; then hadoop_debug "Running ${plugin}_tests" #shellcheck disable=SC2086 ${plugin}_tests (( RESULT = RESULT + $? )) fi done } ## @description Import content from test-patch.d and optionally ## @description from user provided plugin directory ## @audience private ## @stability evolving ## @replaceable no function importplugins { local i local files=() if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then if [[ -d "${BINDIR}/test-patch.d" ]]; then files=(${BINDIR}/test-patch.d/*.sh) fi fi if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}" files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh) fi for i in "${files[@]}"; do hadoop_debug "Importing ${i}" . "${i}" done } ## @description Register test-patch.d plugins ## @audience public ## @stability stable ## @replaceable no function add_plugin { PLUGINS="${PLUGINS} $1" } ############################################################################### ############################################################################### ############################################################################### big_console_header "Bootstrapping test harness" setup_defaults parse_args "$@" importplugins locate_patch find_changed_files determine_needed_tests # from here on out, we'll be in ${BASEDIR} for cwd # routines need to pushd/popd if they change. git_checkout RESULT=$? if [[ ${JENKINS} == "true" ]] ; then if [[ ${RESULT} != 0 ]] ; then exit 100 fi fi check_reexec postcheckout find_changed_modules preapply apply_patch_file postapply check_mvn_install postinstall runtests close_jira_footer close_jira_table output_to_console ${RESULT} output_to_jira ${RESULT} cleanup_and_exit ${RESULT}