diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
index ebcb660c511..ddfd940064f 100755
--- a/dev-support/smart-apply-patch.sh
+++ b/dev-support/smart-apply-patch.sh
@@ -43,7 +43,9 @@ PATCH=${PATCH:-patch} # allow overriding patch binary
# Cleanup handler for temporary files
TOCLEAN=""
cleanup() {
- rm $TOCLEAN
+ if [[ -n ${TOCLEAN} ]]; then
+ rm $TOCLEAN
+ fi
exit $1
}
trap "cleanup 1" HUP INT QUIT TERM
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
index 63115842b39..1fbe88e920e 100755
--- a/dev-support/test-patch.d/checkstyle.sh
+++ b/dev-support/test-patch.d/checkstyle.sh
@@ -18,79 +18,126 @@ add_plugin checkstyle
CHECKSTYLE_TIMER=0
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
function checkstyle_filefilter
{
local filename=$1
- if [[ ${filename} =~ \.java$ ]]; then
- add_test checkstyle
+ if [[ ${BUILDTOOL} == maven
+ || ${BUILDTOOL} == ant ]]; then
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test checkstyle
+ fi
fi
}
-function checkstyle_mvnrunner
+function checkstyle_runner
{
- local logfile=$1
- local output=$2
+ local repostatus=$1
local tmp=${PATCH_DIR}/$$.${RANDOM}
local j
+ local i=0
+ local fn
+ local savestart=${TIMER}
+ local savestop
+ local output
+ local logfile
+ local repo
+ local modulesuffix
+ local cmd
- "${MVN}" clean test checkstyle:checkstyle -DskipTests \
- -Dcheckstyle.consoleOutput=true \
- "-D${PROJECT_NAME}PatchProcess" 2>&1 \
- | tee "${logfile}" \
- | ${GREP} ^/ \
- | ${SED} -e "s,${BASEDIR},.,g" \
- > "${tmp}"
+ modules_reset
- # the checkstyle output files are massive, so
- # let's reduce the work by filtering out files
- # that weren't changed. Some modules are
- # MASSIVE and this can cut the output down to
- # by orders of magnitude!!
- for j in ${CHANGED_FILES}; do
- ${GREP} "${j}" "${tmp}" >> "${output}"
+ if [[ ${repostatus} == branch ]]; then
+ repo=${PATCH_BRANCH}
+ else
+ repo="the patch"
+ fi
+
+ #shellcheck disable=SC2153
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ start_clock
+ fn=$(module_file_fragment "${MODULE[${i}]}")
+ modulesuffix=$(basename "${MODULE[${i}]}")
+ output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt"
+ logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt"
+ pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null
+
+ case ${BUILDTOOL} in
+ maven)
+ cmd="${MVN} ${MAVEN_ARGS[*]} clean test \
+ checkstyle:checkstyle \
+ -Dcheckstyle.consoleOutput=true \
+ ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch"
+ ;;
+ ant)
+ cmd="${ANT} \
+ -Dcheckstyle.consoleOutput=true \
+ ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \
+ ${ANT_ARGS[*]} checkstyle"
+ ;;
+ esac
+
+ #shellcheck disable=SC2086
+ echo ${cmd} "> ${logfile}"
+ #shellcheck disable=SC2086
+ ${cmd} 2>&1 \
+ | tee "${logfile}" \
+ | ${GREP} ^/ \
+ | ${SED} -e "s,${BASEDIR},.,g" \
+ > "${tmp}"
+
+ if [[ $? == 0 ]] ; then
+ module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle"
+ else
+ module_status ${i} -1 "${logfile}" "${modulesuffix} in ${repo} failed checkstyle"
+ ((result = result + 1))
+ fi
+ savestop=$(stop_clock)
+ #shellcheck disable=SC2034
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+
+ for j in ${CHANGED_FILES}; do
+ ${GREP} "${j}" "${tmp}" >> "${output}"
+ done
+
+ rm "${tmp}" 2>/dev/null
+ # shellcheck disable=SC2086
+ popd >/dev/null
+ ((i=i+1))
done
- rm "${tmp}" 2>/dev/null
+ TIMER=${savestart}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
}
function checkstyle_preapply
{
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
+ local result
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: prepatch"
+ big_console_header "${PATCH_BRANCH} checkstyle"
start_clock
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
+ verify_needed_test checkstyle
+ if [[ $? == 0 ]]; then
+ echo "Patch does not need checkstyle testing."
+ return 0
+ fi
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
- add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
- fi
- popd >/dev/null
- done
+ personality_modules branch checkstyle
+ checkstyle_runner branch
+ result=$?
+ modules_messages branch checkstyle true
# keep track of how much as elapsed for us already
CHECKSTYLE_TIMER=$(stop_clock)
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
return 0
}
@@ -135,71 +182,71 @@ function checkstyle_calcdiffs
function checkstyle_postapply
{
- local rc=0
+ local result
local module
- local modules=${CHANGED_MODULES}
- local module_suffix
+ local fn
+ local i=0
local numprepatch=0
local numpostpatch=0
local diffpostpatch=0
- verify_needed_test checkstyle
+ big_console_header "Patch checkstyle plugin"
+ start_clock
+
+ verify_needed_test checkstyle
if [[ $? == 0 ]]; then
+ echo "Patch does not need checkstyle testing."
return 0
fi
- big_console_header "checkstyle plugin: postpatch"
+ personality_modules patch checkstyle
+ checkstyle_runner patch
+ result=$?
- start_clock
# add our previous elapsed to our new timer
# by setting the clock back
offset_clock "${CHECKSTYLE_TIMER}"
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch checkstyle compilation is broken."
- add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
continue
fi
+ module=${MODULE[$i]}
+ fn=$(module_file_fragment "${module}")
+
+ if [[ ! -f "${PATCH_DIR}/branch-checkstyle-${fn}.txt" ]]; then
+ touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt"
+ fi
#shellcheck disable=SC2016
diffpostpatch=$(checkstyle_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
- "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
+ "${PATCH_DIR}/branch-checkstyle-${fn}.txt" \
+ "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \
+ "${PATCH_DIR}/diff-checkstyle-${fn}.txt" )
if [[ ${diffpostpatch} -gt 0 ]] ; then
- ((rc = rc + 1))
+ ((result = result + 1))
# shellcheck disable=SC2016
- numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
+ numprepatch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}')
# shellcheck disable=SC2016
- numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
+ numpostpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}')
- add_jira_table -1 checkstyle "The applied patch generated "\
- "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
- footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
+ module_status ${i} -1 "diff-checkstyle-${fn}.txt" "Patch generated "\
+ "${diffpostpatch} new checkstyle issues in "\
+ "${module} (total was ${numprepatch}, now ${numpostpatch})."
fi
-
- popd >/dev/null
+ ((i=i+1))
done
- if [[ ${rc} -gt 0 ]] ; then
- add_jira_footer checkstyle "${footer}"
+ modules_messages patch checkstyle true
+
+ if [[ ${result} != 0 ]]; then
return 1
fi
- add_jira_table +1 checkstyle "There were no new checkstyle issues."
return 0
-}
\ No newline at end of file
+}
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
index 9277ea50aa4..1c06a5d5e11 100755
--- a/dev-support/test-patch.d/shellcheck.sh
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -56,6 +56,7 @@ function shellcheck_private_findbash
function shellcheck_preapply
{
local i
+ local msg
verify_needed_test shellcheck
if [[ $? == 0 ]]; then
@@ -65,7 +66,7 @@ function shellcheck_preapply
big_console_header "shellcheck plugin: prepatch"
if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
+ yetus_error "shellcheck is not available."
return 0
fi
@@ -73,6 +74,13 @@ function shellcheck_preapply
# shellcheck disable=SC2016
SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
+ msg="v${SHELLCHECK_VERSION}"
+
+ if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then
+ msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)"
+ fi
+
+ add_footer_table shellcheck "${msg}"
echo "Running shellcheck against all identifiable shell scripts"
pushd "${BASEDIR}" >/dev/null
@@ -138,8 +146,8 @@ function shellcheck_postapply
big_console_header "shellcheck plugin: postpatch"
if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- add_jira_table 0 shellcheck "Shellcheck was not available."
+ yetus_error "shellcheck is not available."
+ add_vote_table 0 shellcheck "Shellcheck was not available."
return 0
fi
@@ -155,8 +163,13 @@ function shellcheck_postapply
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
done
+ if [[ ! -f "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" ]]; then
+ touch "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
+ fi
+
# shellcheck disable=SC2016
numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
+
# shellcheck disable=SC2016
numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
@@ -167,12 +180,12 @@ function shellcheck_postapply
)
if [[ ${diffPostpatch} -gt 0 ]] ; then
- add_jira_table -1 shellcheck "The applied patch generated "\
+ add_vote_table -1 shellcheck "The applied patch generated "\
"${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
- add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
+ add_footer_table shellcheck "@@BASE@@/diffpatchshellcheck.txt"
return 1
fi
- add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
+ add_vote_table +1 shellcheck "There were no new shellcheck issues."
return 0
}
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
index 324481ca74d..6fc033b3f31 100755
--- a/dev-support/test-patch.d/whitespace.sh
+++ b/dev-support/test-patch.d/whitespace.sh
@@ -19,28 +19,47 @@ add_plugin whitespace
function whitespace_postapply
{
local count
- local j
+ local result=0
big_console_header "Checking for whitespace at the end of lines"
start_clock
pushd "${BASEDIR}" >/dev/null
- for j in ${CHANGED_FILES}; do
- ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
- done
+ # shellcheck disable=SC2016
+ ${AWK} '/\t/ {print $0}' \
+ "${GITDIFFCONTENT}" \
+ | ${GREP} -v Makefile: >> "${PATCH_DIR}/whitespace-tabs.txt"
+
+ ${GREP} -E '[[:blank:]]$' \
+ "${GITDIFFCONTENT}" \
+ >> "${PATCH_DIR}/whitespace-eol.txt"
# shellcheck disable=SC2016
- count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
+ count=$(wc -l "${PATCH_DIR}/whitespace-eol.txt" | ${AWK} '{print $1}')
if [[ ${count} -gt 0 ]]; then
- add_jira_table -1 whitespace "The patch has ${count}"\
+ add_vote_table -1 whitespace "The patch has ${count}"\
" line(s) that end in whitespace. Use git apply --whitespace=fix."
- add_jira_footer whitespace "@@BASE@@/whitespace.txt"
+ add_footer_table whitespace "@@BASE@@/whitespace-eol.txt"
+ ((result=result+1))
+ fi
+
+ # shellcheck disable=SC2016
+ count=$(wc -l "${PATCH_DIR}/whitespace-tabs.txt" | ${AWK} '{print $1}')
+
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 whitespace "The patch has ${count}"\
+ " line(s) with tabs."
+ add_footer_table whitespace "@@BASE@@/whitespace-tabs.txt"
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
popd >/dev/null
return 1
fi
popd >/dev/null
- add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
+ add_vote_table +1 whitespace "Patch has no whitespace issues."
return 0
}
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index cd91a5c8cd8..e1dadd282c8 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -14,14 +14,41 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+ || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+ echo "bash v3.2+ is required. Sorry."
+ exit 1
+fi
+
### BUILD_URL is set by Hudson if it is run by patch process
this="${BASH_SOURCE-$0}"
BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-CWD=$(pwd)
+STARTINGDIR=$(pwd)
USER_PARAMS=("$@")
GLOBALTIMER=$(date +"%s")
+# global arrays
+declare -a MAVEN_ARGS=("--batch-mode")
+declare -a ANT_ARGS=("-noinput")
+declare -a TP_HEADER
+declare -a TP_VOTE_TABLE
+declare -a TP_TEST_TABLE
+declare -a TP_FOOTER_TABLE
+declare -a MODULE_STATUS
+declare -a MODULE_STATUS_TIMER
+declare -a MODULE_STATUS_MSG
+declare -a MODULE_STATUS_LOG
+declare -a MODULE
+
+TP_HEADER_COUNTER=0
+TP_VOTE_COUNTER=0
+TP_TEST_COUNTER=0
+TP_FOOTER_COUNTER=0
+
## @description Setup the default global variables
## @audience public
## @stability stable
@@ -33,10 +60,15 @@ function setup_defaults
else
MVN=${MAVEN_HOME}/bin/mvn
fi
- # This parameter needs to be kept as an array
- MAVEN_ARGS=()
- PROJECT_NAME=hadoop
+ if [[ -z "${ANT_HOME:-}" ]]; then
+ ANT=ant
+ else
+ ANT=${ANT_HOME}/bin/ant
+ fi
+
+ PROJECT_NAME=yetus
+ DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub"
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
JENKINS=false
BASEDIR=$(pwd)
@@ -44,13 +76,15 @@ function setup_defaults
USER_PLUGIN_DIR=""
LOAD_SYSTEM_PLUGINS=true
+ ALLOWSUMMARIES=true
- FINDBUGS_HOME=${FINDBUGS_HOME:-}
- FINDBUGS_WARNINGS_FAIL_PRECHECK=false
+ DOCKERSUPPORT=false
ECLIPSE_HOME=${ECLIPSE_HOME:-}
BUILD_NATIVE=${BUILD_NATIVE:-true}
PATCH_BRANCH=""
- PATCH_BRANCH_DEFAULT="trunk"
+ PATCH_BRANCH_DEFAULT="master"
+
+ #shellcheck disable=SC2034
CHANGED_MODULES=""
USER_MODULE_LIST=""
OFFLINE=false
@@ -58,50 +92,40 @@ function setup_defaults
REEXECED=false
RESETREPO=false
ISSUE=""
- ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
+ ISSUE_RE='^(YETUS)-[0-9]+$'
TIMER=$(date +"%s")
PATCHURL=""
-
OSTYPE=$(uname -s)
+ BUILDTOOL=maven
+ BUGSYSTEM=jira
+ JDK_TEST_LIST="javac javadoc unit"
+ GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt"
+ GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
# Solaris needs POSIX, not SVID
case ${OSTYPE} in
SunOS)
- PS=${PS:-ps}
AWK=${AWK:-/usr/xpg4/bin/awk}
SED=${SED:-/usr/xpg4/bin/sed}
WGET=${WGET:-wget}
GIT=${GIT:-git}
- EGREP=${EGREP:-/usr/xpg4/bin/egrep}
GREP=${GREP:-/usr/xpg4/bin/grep}
- PATCH=${PATCH:-patch}
+ PATCH=${PATCH:-/usr/gnu/bin/patch}
DIFF=${DIFF:-/usr/gnu/bin/diff}
- JIRACLI=${JIRA:-jira}
FILE=${FILE:-file}
;;
*)
- PS=${PS:-ps}
AWK=${AWK:-awk}
SED=${SED:-sed}
WGET=${WGET:-wget}
GIT=${GIT:-git}
- EGREP=${EGREP:-egrep}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
DIFF=${DIFF:-diff}
- JIRACLI=${JIRA:-jira}
FILE=${FILE:-file}
;;
esac
- declare -a JIRA_COMMENT_TABLE
- declare -a JIRA_FOOTER_TABLE
- declare -a JIRA_HEADER
- declare -a JIRA_TEST_TABLE
-
- JFC=0
- JTC=0
- JTT=0
RESULT=0
}
@@ -110,7 +134,7 @@ function setup_defaults
## @stability stable
## @replaceable no
## @param string
-function hadoop_error
+function yetus_error
{
echo "$*" 1>&2
}
@@ -120,20 +144,51 @@ function hadoop_error
## @stability stable
## @replaceable no
## @param string
-function hadoop_debug
+function yetus_debug
{
- if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
+ if [[ -n "${TP_SHELL_SCRIPT_DEBUG}" ]]; then
echo "[$(date) DEBUG]: $*" 1>&2
fi
}
+## @description Convert the given module name to a file fragment
+## @audience public
+## @stability stable
+## @replaceable no
+## @param module
+function module_file_fragment
+{
+ local mod=$1
+ if [[ ${mod} == . ]]; then
+ echo root
+ else
+ echo "$1" | tr '/' '_' | tr '\\' '_'
+ fi
+}
+
+## @description Convert time in seconds to m + s
+## @audience public
+## @stability stable
+## @replaceable no
+## @param seconds
+function clock_display
+{
+ local -r elapsed=$1
+
+ if [[ ${elapsed} -lt 0 ]]; then
+ echo "N/A"
+ else
+ printf "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
+ fi
+}
+
## @description Activate the local timer
## @audience public
## @stability stable
## @replaceable no
function start_clock
{
- hadoop_debug "Start clock"
+ yetus_debug "Start clock"
TIMER=$(date +"%s")
}
@@ -145,7 +200,7 @@ function stop_clock
{
local -r stoptime=$(date +"%s")
local -r elapsed=$((stoptime-TIMER))
- hadoop_debug "Stop clock"
+ yetus_debug "Stop clock"
echo ${elapsed}
}
@@ -158,7 +213,7 @@ function stop_global_clock
{
local -r stoptime=$(date +"%s")
local -r elapsed=$((stoptime-GLOBALTIMER))
- hadoop_debug "Stop global clock"
+ yetus_debug "Stop global clock"
echo ${elapsed}
}
@@ -178,10 +233,10 @@ function offset_clock
## @stability stable
## @replaceable no
## @param string
-function add_jira_header
+function add_header_line
{
- JIRA_HEADER[${JHC}]="| $* |"
- JHC=$(( JHC+1 ))
+ TP_HEADER[${TP_HEADER_COUNTER}]="$*"
+ ((TP_HEADER_COUNTER=TP_HEADER_COUNTER+1 ))
}
## @description Add to the output table. If the first parameter is a number
@@ -199,88 +254,152 @@ function add_jira_header
## @param subsystem
## @param string
## @return Elapsed time display
-function add_jira_table
+function add_vote_table
{
local value=$1
local subsystem=$2
shift 2
- local color
- local calctime=0
-
+ local calctime
local -r elapsed=$(stop_clock)
- if [[ ${elapsed} -lt 0 ]]; then
- calctime="N/A"
- else
- printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
+ yetus_debug "add_vote_table ${value} ${subsystem} ${*}"
+
+ calctime=$(clock_display "${elapsed}")
+
+ if [[ ${value} == "1" ]]; then
+ value="+1"
fi
- echo ""
- echo "Elapsed time: ${calctime}"
- echo ""
-
- case ${value} in
- 1|+1)
- value="+1"
- color="green"
- ;;
- -1)
- color="red"
- ;;
- 0)
- color="blue"
- ;;
- null)
- ;;
- esac
-
- if [[ -z ${color} ]]; then
- JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |"
- JTC=$(( JTC+1 ))
+ if [[ -z ${value} ]]; then
+ TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | ${subsystem} | | ${*:-} |"
else
- JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |"
- JTC=$(( JTC+1 ))
+ TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |"
fi
+ ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1))
}
-## @description Put the final environment information at the bottom
+## @description Report the JVM version of the given directory
+## @stability stable
+## @audience private
+## @replaceable yes
+## @params directory
+## @returns version
+function report_jvm_version
+{
+ #shellcheck disable=SC2016
+ "${1}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \"
+}
+
+## @description Verify if a given test is multijdk
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param test
+## @return 1 = yes
+## @return 0 = no
+function verify_multijdk_test
+{
+ local i=$1
+
+ if [[ "${JDK_DIR_LIST}" == "${JAVA_HOME}" ]]; then
+ yetus_debug "MultiJDK not configured."
+ return 0
+ fi
+
+ if [[ ${JDK_TEST_LIST} =~ $i ]]; then
+ yetus_debug "${i} is in ${JDK_TEST_LIST} and MultiJDK configured."
+ return 1
+ fi
+ return 0
+}
+
+## @description Absolute path the JDK_DIR_LIST and JAVA_HOME.
+## @description if JAVA_HOME is in JDK_DIR_LIST, it is positioned last
+## @stability stable
+## @audience private
+## @replaceable yes
+function fullyqualifyjdks
+{
+ local i
+ local jdkdir
+ local tmplist
+
+ JAVA_HOME=$(cd -P -- "${JAVA_HOME}" >/dev/null && pwd -P)
+
+ for i in ${JDK_DIR_LIST}; do
+ jdkdir=$(cd -P -- "${i}" >/dev/null && pwd -P)
+ if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then
+ tmplist="${tmplist} ${jdkdir}"
+ fi
+ done
+
+ JDK_DIR_LIST="${tmplist} ${JAVA_HOME}"
+ JDK_DIR_LIST=${JDK_DIR_LIST/ }
+}
+
+## @description Put the opening environment information at the bottom
## @description of the footer table
## @stability stable
## @audience private
## @replaceable yes
-function close_jira_footer
+function prepopulate_footer
{
# shellcheck disable=SC2016
- local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \")
+ local javaversion
+ local listofjdks
local -r unamea=$(uname -a)
+ local i
- add_jira_footer "Java" "${javaversion}"
- add_jira_footer "uname" "${unamea}"
+ add_footer_table "uname" "${unamea}"
+ add_footer_table "Build tool" "${BUILDTOOL}"
+
+ if [[ -n ${PERSONALITY} ]]; then
+ add_footer_table "Personality" "${PERSONALITY}"
+ fi
+
+ javaversion=$(report_jvm_version "${JAVA_HOME}")
+ add_footer_table "Default Java" "${javaversion}"
+ if [[ -n ${JDK_DIR_LIST}
+ && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then
+ for i in ${JDK_DIR_LIST}; do
+ javaversion=$(report_jvm_version "${i}")
+ listofjdks="${listofjdks} ${i}:${javaversion}"
+ done
+ add_footer_table "Multi-JDK versions" "${listofjdks}"
+ fi
+}
+
+## @description Put docker stats in various tables
+## @stability stable
+## @audience private
+## @replaceable yes
+function finish_docker_stats
+{
+ if [[ ${DOCKERMODE} == true ]]; then
+ # DOCKER_VERSION is set by our creator.
+ add_footer_table "Docker" "${DOCKER_VERSION}"
+ fi
}
## @description Put the final elapsed time at the bottom of the table.
## @audience private
## @stability stable
## @replaceable no
-function close_jira_table
+function finish_vote_table
{
local -r elapsed=$(stop_global_clock)
+ local calctime
- if [[ ${elapsed} -lt 0 ]]; then
- calctime="N/A"
- else
- printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
- fi
+ calctime=$(clock_display "${elapsed}")
echo ""
echo "Total Elapsed time: ${calctime}"
echo ""
-
- JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |"
- JTC=$(( JTC+1 ))
+ TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${calctime} | |"
+ ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1 ))
}
## @description Add to the footer of the display. @@BASE@@ will get replaced with the
@@ -291,13 +410,13 @@ function close_jira_table
## @replaceable no
## @param subsystem
## @param string
-function add_jira_footer
+function add_footer_table
{
local subsystem=$1
shift 1
- JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |"
- JFC=$(( JFC+1 ))
+ TP_FOOTER_TABLE[${TP_FOOTER_COUNTER}]="| ${subsystem} | $* |"
+ ((TP_FOOTER_COUNTER=TP_FOOTER_COUNTER+1 ))
}
## @description Special table just for unit test failures
@@ -306,13 +425,13 @@ function add_jira_footer
## @replaceable no
## @param failurereason
## @param testlist
-function add_jira_test_table
+function add_test_table
{
local failure=$1
shift 1
- JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |"
- JTT=$(( JTT+1 ))
+ TP_TEST_TABLE[${TP_TEST_COUNTER}]="| ${failure} | $* |"
+ ((TP_TEST_COUNTER=TP_TEST_COUNTER+1 ))
}
## @description Large display for the user console
@@ -334,29 +453,6 @@ function big_console_header
printf "\n\n"
}
-## @description Remove {color} tags from a string
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-## @return string
-function colorstripper
-{
- local string=$1
- shift 1
-
- local green=""
- local white=""
- local red=""
- local blue=""
-
- echo "${string}" | \
- ${SED} -e "s,{color:red},${red},g" \
- -e "s,{color:green},${green},g" \
- -e "s,{color:blue},${blue},g" \
- -e "s,{color},${white},g"
-}
-
## @description Find the largest size of a column of an array
## @audience private
## @stability evolving
@@ -391,7 +487,7 @@ function find_java_home
{
start_clock
if [[ -z ${JAVA_HOME:-} ]]; then
- case $(uname -s) in
+ case ${OSTYPE} in
Darwin)
if [[ -z "${JAVA_HOME}" ]]; then
if [[ -x /usr/libexec/java_home ]]; then
@@ -409,7 +505,7 @@ function find_java_home
if [[ -z ${JAVA_HOME:-} ]]; then
echo "JAVA_HOME is not defined."
- add_jira_table -1 pre-patch "JAVA_HOME is not defined."
+ add_vote_table -1 pre-patch "JAVA_HOME is not defined."
return 1
fi
return 0
@@ -420,26 +516,17 @@ function find_java_home
## @stability stable
## @audience public
## @returns ${JIRACLI} exit code
-function write_to_jira
+function write_comment
{
local -r commentfile=${1}
shift
- local retval
+ local retval=0
if [[ ${OFFLINE} == false
&& ${JENKINS} == true ]]; then
- export USER=hudson
- # shellcheck disable=SC2086
- ${JIRACLI} --comment "$(cat ${commentfile})" \
- -s https://issues.apache.org/jira \
- -a addcomment -u hadoopqa \
- -p "${JIRA_PASSWD}" \
- --issue "${ISSUE}"
+ ${BUGSYSTEM}_write_comment "${commentfile}"
retval=$?
- ${JIRACLI} -s https://issues.apache.org/jira \
- -a logout -u hadoopqa \
- -p "${JIRA_PASSWD}"
fi
return ${retval}
}
@@ -457,44 +544,46 @@ function verify_patchdir_still_exists
local extra=""
if [[ ! -d ${PATCH_DIR} ]]; then
- rm "${commentfile}" 2>/dev/null
+ rm "${commentfile}" 2>/dev/null
- echo "(!) The patch artifact directory has been removed! " > "${commentfile}"
- echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}"
- echo
- cat ${commentfile}
- echo
- if [[ ${JENKINS} == true ]]; then
- if [[ -n ${NODE_NAME} ]]; then
- extra=" (node ${NODE_NAME})"
- fi
- echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}"
-
- write_to_jira ${commentfile}
+ echo "(!) The patch artifact directory has been removed! " > "${commentfile}"
+ echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}"
+ echo
+ cat ${commentfile}
+ echo
+ if [[ ${JENKINS} == true ]]; then
+ if [[ -n ${NODE_NAME} ]]; then
+ extra=" (node ${NODE_NAME})"
fi
+ echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}"
- rm "${commentfile}"
- cleanup_and_exit ${RESULT}
+ write_comment ${commentfile}
fi
+
+ rm "${commentfile}"
+ cleanup_and_exit ${RESULT}
+ fi
}
-## @description generate a list of all files and line numbers that
-## @description that were added/changed in the source repo
+## @description generate a list of all files and line numbers in $GITDIFFLINES that
+## @description that were added/changed in the source repo. $GITDIFFCONTENT
+## @description is same file, but also includes the content of those lines
## @audience private
## @stability stable
-## @params filename
## @replaceable no
function compute_gitdiff
{
- local outfile=$1
local file
local line
local startline
local counter
local numlines
local actual
+ local content
+ local outfile="${PATCH_DIR}/computegitdiff.${RANDOM}"
pushd "${BASEDIR}" >/dev/null
+ ${GIT} add --all --intent-to-add
while read line; do
if [[ ${line} =~ ^\+\+\+ ]]; then
file="./"$(echo "${line}" | cut -f2- -d/)
@@ -511,11 +600,27 @@ function compute_gitdiff
numlines=1
fi
counter=0
- until [[ ${counter} -gt ${numlines} ]]; do
+ # it isn't obvious, but on MOST platforms under MOST use cases,
+ # this is faster than using sed, and definitely faster than using
+ # awk.
+ # http://unix.stackexchange.com/questions/47407/cat-line-x-to-line-y-on-a-huge-file
+ # has a good discussion w/benchmarks
+ #
+ # note that if tail is still sending data through the pipe, but head gets enough
+ # to do what was requested, head will exit, leaving tail with a broken pipe.
+ # we're going to send stderr to /dev/null and ignore the error since head's
+ # output is really what we're looking for
+ tail -n "+${startline}" "${file}" 2>/dev/null | head -n ${numlines} > "${outfile}"
+ oldifs=${IFS}
+ IFS=''
+ while read -r content; do
((actual=counter+startline))
- echo "${file}:${actual}:" >> "${outfile}"
+ echo "${file}:${actual}:" >> "${GITDIFFLINES}"
+ printf "%s:%s:%s\n" "${file}" "${actual}" "${content}" >> "${GITDIFFCONTENT}"
((counter=counter+1))
- done
+ done < "${outfile}"
+ rm "${outfile}"
+ IFS=${oldifs}
fi
done < <("${GIT}" diff --unified=0 --no-color)
popd >/dev/null
@@ -540,35 +645,39 @@ function echo_and_redirect
verify_patchdir_still_exists
find "${BASEDIR}" -type d -exec chmod +x {} \;
+ # to the screen
echo "${*} > ${logfile} 2>&1"
- "${@}" > "${logfile}" 2>&1
+ # to the log
+ echo "${*}" > "${logfile}"
+ # the actual command
+ "${@}" >> "${logfile}" 2>&1
}
-## @description is PATCH_DIR relative to BASEDIR?
+## @description is a given directory relative to BASEDIR?
## @audience public
## @stability stable
## @replaceable yes
-## @returns 1 - no, PATCH_DIR
-## @returns 0 - yes, PATCH_DIR - BASEDIR
-function relative_patchdir
+## @param path
+## @returns 1 - no, path
+## @returns 0 - yes, path - BASEDIR
+function relative_dir
{
- local p=${PATCH_DIR#${BASEDIR}}
+ local p=${1#${BASEDIR}}
- if [[ ${#p} -eq ${#PATCH_DIR} ]]; then
- echo ${p}
+ if [[ ${#p} -eq ${#1} ]]; then
+ echo "${p}"
return 1
fi
p=${p#/}
- echo ${p}
+ echo "${p}"
return 0
}
-
## @description Print the usage information
## @audience public
## @stability stable
## @replaceable no
-function hadoop_usage
+function testpatch_usage
{
local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]')
@@ -582,25 +691,36 @@ function hadoop_usage
echo "Options:"
echo "--basedir=
The directory to apply the patch to (default current directory)"
echo "--branch=[ Forcibly set the branch"
- echo "--branch-default=][ If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')"
+ echo "--branch-default=][ If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'master')"
+ #not quite working yet
+ #echo "--bugsystem= The bug system in use ('jira', the default, or 'github')"
echo "--build-native= If true, then build native components (default 'true')"
- echo "--contrib-guide= URL to point new users towards project conventions. (default Hadoop's wiki)"
+ echo "--build-tool= Pick which build tool to focus around (maven, ant)"
+ echo "--contrib-guide= URL to point new users towards project conventions. (default: ${HOW_TO_CONTRIBUTE} )"
echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
- echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)"
- echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
- echo "--issue-re= Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
+ echo "--docker Spawn a docker container"
+ echo "--dockerfile= Dockerfile fragment to use as the base"
+ echo "--issue-re= Bash regular expression to use when trying to find a jira ref in the patch name (default: \'${ISSUE_RE}\')"
+ echo "--java-home= Set JAVA_HOME (In Docker mode, this should be local to the image)"
+ echo "--multijdkdirs= Comma delimited lists of JDK paths to use for multi-JDK tests"
+ echo "--multijdktests=] Comma delimited tests to use when multijdkdirs is used. (default: javac,javadoc,unit)"
echo "--modulelist= Specify additional modules to test (comma delimited)"
echo "--offline Avoid connecting to the Internet"
- echo "--patch-dir= The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')"
+ echo "--patch-dir= The directory for working and output files (default '/tmp/test-patch-${PROJECT_NAME}/pid')"
+ echo "--personality= The personality file to load"
echo "--plugins= A directory of user provided plugins. see test-patch.d for examples (default empty)"
- echo "--project= The short name for project currently using test-patch (default 'hadoop')"
+ echo "--project= The short name for project currently using test-patch (default 'yetus')"
echo "--resetrepo Forcibly clean the repo"
echo "--run-tests Run all relevant tests below the base directory"
echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d"
+ echo "--summarize= Allow tests to summarize results"
echo "--testlist= Specify which subsystem tests to use (comma delimited)"
-
+ echo "--test-parallel= Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)"
+ echo "--test-threads= Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
+ echo ""
echo "Shell binary overrides:"
+ echo "--ant-cmd= The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')"
echo "--awk-cmd= The 'awk' command to use (default 'awk')"
echo "--diff-cmd= The GNU-compatible 'diff' command to use (default 'diff')"
echo "--file-cmd= The 'file' command to use (default 'file')"
@@ -608,17 +728,24 @@ function hadoop_usage
echo "--grep-cmd= The 'grep' command to use (default 'grep')"
echo "--mvn-cmd= The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
echo "--patch-cmd= The 'patch' command to use (default 'patch')"
- echo "--ps-cmd= The 'ps' command to use (default 'ps')"
echo "--sed-cmd= The 'sed' command to use (default 'sed')"
echo
echo "Jenkins-only options:"
echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
+ echo "--build-url Set the build location web page"
echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)"
- echo "--jira-cmd= The 'jira' command to use (default 'jira')"
- echo "--jira-password= The password for the 'jira' command"
echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
echo "--wget-cmd= The 'wget' command to use (default 'wget')"
+
+ importplugins
+
+ for plugin in ${PLUGINS} ${BUGSYSTEMS}; do
+ if declare -f ${plugin}_usage >/dev/null 2>&1; then
+ echo
+ "${plugin}_usage"
+ fi
+ done
}
## @description Interpret the command line parameters
@@ -634,6 +761,9 @@ function parse_args
for i in "$@"; do
case ${i} in
+ --ant-cmd=*)
+ ANT=${i#*=}
+ ;;
--awk-cmd=*)
AWK=${i#*=}
;;
@@ -646,14 +776,23 @@ function parse_args
--branch-default=*)
PATCH_BRANCH_DEFAULT=${i#*=}
;;
+ --bugsystem=*)
+ BUGSYSTEM=${i#*=}
+ ;;
--build-native=*)
BUILD_NATIVE=${i#*=}
;;
+ --build-tool=*)
+ BUILDTOOL=${i#*=}
+ ;;
+ --build-url=*)
+ BUILD_URL=${i#*=}
+ ;;
--contrib-guide=*)
HOW_TO_CONTRIBUTE=${i#*=}
;;
--debug)
- HADOOP_SHELL_SCRIPT_DEBUG=true
+ TP_SHELL_SCRIPT_DEBUG=true
;;
--diff-cmd=*)
DIFF=${i#*=}
@@ -661,18 +800,21 @@ function parse_args
--dirty-workspace)
DIRTY_WORKSPACE=true
;;
+ --docker)
+ DOCKERSUPPORT=true
+ ;;
+ --dockerfile=*)
+ DOCKERFILE=${i#*=}
+ ;;
+ --dockermode)
+ DOCKERMODE=true
+ ;;
--eclipse-home=*)
ECLIPSE_HOME=${i#*=}
;;
--file-cmd=*)
FILE=${i#*=}
;;
- --findbugs-home=*)
- FINDBUGS_HOME=${i#*=}
- ;;
- --findbugs-strict-precheck)
- FINDBUGS_WARNINGS_FAIL_PRECHECK=true
- ;;
--git-cmd=*)
GIT=${i#*=}
;;
@@ -680,7 +822,7 @@ function parse_args
GREP=${i#*=}
;;
--help|-help|-h|help|--h|--\?|-\?|\?)
- hadoop_usage
+ testpatch_usage
exit 0
;;
--issue-re=*)
@@ -691,17 +833,22 @@ function parse_args
;;
--jenkins)
JENKINS=true
- ;;
- --jira-cmd=*)
- JIRACLI=${i#*=}
- ;;
- --jira-password=*)
- JIRA_PASSWD=${i#*=}
+ TEST_PARALLEL=${TEST_PARALLEL:-true}
;;
--modulelist=*)
USER_MODULE_LIST=${i#*=}
USER_MODULE_LIST=${USER_MODULE_LIST//,/ }
- hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}"
+ yetus_debug "Manually forcing modules ${USER_MODULE_LIST}"
+ ;;
+ --multijdkdirs=*)
+ JDK_DIR_LIST=${i#*=}
+ JDK_DIR_LIST=${JDK_DIR_LIST//,/ }
+ yetus_debug "Multi-JVM mode activated with ${JDK_DIR_LIST}"
+ ;;
+ --multijdktests=*)
+ JDK_TEST_LIST=${i#*=}
+ JDK_TEST_LIST=${JDK_TEST_LIST//,/ }
+ yetus_debug "Multi-JVM test list: ${JDK_TEST_LIST}"
;;
--mvn-cmd=*)
MVN=${i#*=}
@@ -718,19 +865,17 @@ function parse_args
--patch-dir=*)
USER_PATCH_DIR=${i#*=}
;;
+ --personality=*)
+ PERSONALITY=${i#*=}
+ ;;
--plugins=*)
USER_PLUGIN_DIR=${i#*=}
;;
--project=*)
PROJECT_NAME=${i#*=}
;;
- --ps-cmd=*)
- PS=${i#*=}
- ;;
--reexec)
REEXECED=true
- start_clock
- add_jira_table 0 reexec "dev-support patch detected."
;;
--resetrepo)
RESETREPO=true
@@ -741,41 +886,95 @@ function parse_args
--skip-system-plugins)
LOAD_SYSTEM_PLUGINS=false
;;
+ --summarize=*)
+ ALLOWSUMMARIES=${i#*=}
+ ;;
--testlist=*)
testlist=${i#*=}
testlist=${testlist//,/ }
for j in ${testlist}; do
- hadoop_debug "Manually adding patch test subsystem ${j}"
+ yetus_debug "Manually adding patch test subsystem ${j}"
add_test "${j}"
done
;;
+ --test-parallel=*)
+ TEST_PARALLEL=${i#*=}
+ ;;
+ --test-threads=*)
+ # shellcheck disable=SC2034
+ TEST_THREADS=${i#*=}
+ ;;
+ --tpglobaltimer=*)
+ GLOBALTIMER=${i#*=}
+ ;;
+ --tpreexectimer=*)
+ REEXECLAUNCHTIMER=${i#*=}
+ ;;
--wget-cmd=*)
WGET=${i#*=}
;;
+ --*)
+ ## PATCH_OR_ISSUE can't be a --. So this is probably
+ ## a plugin thing.
+ continue
+ ;;
*)
PATCH_OR_ISSUE=${i}
;;
esac
done
+ if [[ -n ${REEXECLAUNCHTIMER} ]]; then
+ TIMER=${REEXECLAUNCHTIMER};
+ else
+ start_clock
+ fi
+
+ if [[ ${REEXECED} == true
+ && ${DOCKERMODE} == true ]]; then
+ add_vote_table 0 reexec "docker + precommit patch detected."
+ elif [[ ${REEXECED} == true ]]; then
+ add_vote_table 0 reexec "precommit patch detected."
+ elif [[ ${DOCKERMODE} == true ]]; then
+ add_vote_table 0 reexec "docker mode."
+ fi
+
# if we requested offline, pass that to mvn
- if [[ ${OFFLINE} == "true" ]] ; then
+ if [[ ${OFFLINE} == "true" ]]; then
MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
+ ANT_ARGS=(${ANT_ARGS[@]} -Doffline=)
+ fi
+
+ if [[ -z "${PATCH_OR_ISSUE}" ]]; then
+ testpatch_usage
+ exit 1
fi
# we need absolute dir for ${BASEDIR}
- cd "${CWD}"
+ cd "${STARTINGDIR}"
BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
- if [[ ${BUILD_NATIVE} == "true" ]] ; then
- NATIVE_PROFILE=-Pnative
- REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
+ if [[ -n ${USER_PATCH_DIR} ]]; then
+ PATCH_DIR="${USER_PATCH_DIR}"
+ else
+ PATCH_DIR=/tmp/test-patch-${PROJECT_NAME}/$$
fi
- if [[ -z "${PATCH_OR_ISSUE}" ]]; then
- hadoop_usage
- exit 1
+
+ cd "${STARTINGDIR}"
+ if [[ ! -d ${PATCH_DIR} ]]; then
+ mkdir -p "${PATCH_DIR}"
+ if [[ $? == 0 ]] ; then
+ echo "${PATCH_DIR} has been created"
+ else
+ echo "Unable to create ${PATCH_DIR}"
+ cleanup_and_exit 1
+ fi
fi
- if [[ ${JENKINS} == "true" ]] ; then
+
+ # we need absolute dir for PATCH_DIR
+ PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
+
+ if [[ ${JENKINS} == "true" ]]; then
echo "Running in Jenkins mode"
ISSUE=${PATCH_OR_ISSUE}
RESETREPO=true
@@ -790,46 +989,61 @@ function parse_args
JENKINS=false
fi
- if [[ -n ${USER_PATCH_DIR} ]]; then
- PATCH_DIR="${USER_PATCH_DIR}"
- else
- PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$
+ if [[ -n "${USER_PLUGIN_DIR}" ]]; then
+ USER_PLUGIN_DIR=$(cd -P -- "${USER_PLUGIN_DIR}" >/dev/null && pwd -P)
fi
- cd "${CWD}"
- if [[ ! -d ${PATCH_DIR} ]]; then
- mkdir -p "${PATCH_DIR}"
- if [[ $? == 0 ]] ; then
- echo "${PATCH_DIR} has been created"
- else
- echo "Unable to create ${PATCH_DIR}"
- cleanup_and_exit 1
- fi
- fi
-
- # we need absolute dir for PATCH_DIR
- PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
-
- GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt
+ GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt"
+ GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
}
## @description Locate the pom.xml file for a given directory
## @audience private
## @stability stable
## @replaceable no
-## @return directory containing the pom.xml
-function find_pom_dir
+## @return directory containing the pom.xml. Nothing returned if not found.
+function find_pomxml_dir
{
local dir
dir=$(dirname "$1")
- hadoop_debug "Find pom dir for: ${dir}"
+ yetus_debug "Find pom.xml dir for: ${dir}"
while builtin true; do
if [[ -f "${dir}/pom.xml" ]];then
echo "${dir}"
- hadoop_debug "Found: ${dir}"
+ yetus_debug "Found: ${dir}"
+ return
+ elif [[ ${dir} == "." ]]; then
+ yetus_error "ERROR: pom.xml is not found. Make sure the target is a Maven-based project."
+ return
+ else
+ dir=$(dirname "${dir}")
+ fi
+ done
+}
+
+## @description Locate the build.xml file for a given directory
+## @audience private
+## @stability stable
+## @replaceable no
+## @return directory containing the build.xml. Nothing returned if not found.
+function find_buildxml_dir
+{
+ local dir
+
+ dir=$(dirname "$1")
+
+ yetus_debug "Find build.xml dir for: ${dir}"
+
+ while builtin true; do
+ if [[ -f "${dir}/build.xml" ]];then
+ echo "${dir}"
+ yetus_debug "Found: ${dir}"
+ return
+ elif [[ ${dir} == "." ]]; then
+ yetus_error "ERROR: build.xml is not found. Make sure the target is a Ant-based project."
return
else
dir=$(dirname "${dir}")
@@ -858,33 +1072,65 @@ function find_changed_files
| sort -u)
}
-## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies
+## @description Find the modules of the build that ${PATCH_DIR}/patch modifies
## @audience private
## @stability stable
## @replaceable no
-## @return None; sets ${CHANGED_MODULES}
+## @return None; sets ${CHANGED_MODULES} and ${CHANGED_UNFILTERED_MODULES}
function find_changed_modules
{
# Come up with a list of changed files into ${TMP}
local pomdirs
+ local pomdir
local module
local pommods
# Now find all the modules that were changed
for file in ${CHANGED_FILES}; do
- #shellcheck disable=SC2086
- pomdirs="${pomdirs} $(find_pom_dir ${file})"
+ case ${BUILDTOOL} in
+ maven)
+ #shellcheck disable=SC2086
+ pomdir=$(find_pomxml_dir ${file})
+ if [[ -z ${pomdir} ]]; then
+ output_to_console 1
+ output_to_bugsystem 1
+ cleanup_and_exit 1
+ fi
+ pomdirs="${pomdirs} ${pomdir}"
+ ;;
+ ant)
+ #shellcheck disable=SC2086
+ pomdir=$(find_buildxml_dir ${file})
+ if [[ -z ${pomdir} ]]; then
+ output_to_console 1
+ output_to_bugsystem 1
+ cleanup_and_exit 1
+ fi
+ pomdirs="${pomdirs} ${pomdir}"
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ output_to_console 1
+ output_to_bugsystem 1
+ cleanup_and_exit 1
+ ;;
+ esac
done
- # Filter out modules without code
- for module in ${pomdirs}; do
- ${GREP} "pom" "${module}/pom.xml" > /dev/null
- if [[ "$?" != 0 ]]; then
- pommods="${pommods} ${module}"
- fi
- done
+ #shellcheck disable=SC2086,SC2034
+ CHANGED_UNFILTERED_MODULES=$(echo ${pomdirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
- #shellcheck disable=SC2086
+ if [[ ${BUILDTOOL} == maven ]]; then
+ # Filter out modules without code
+ for module in ${pomdirs}; do
+ ${GREP} "pom" "${module}/pom.xml" > /dev/null
+ if [[ "$?" != 0 ]]; then
+ pommods="${pommods} ${module}"
+ fi
+ done
+ fi
+
+ #shellcheck disable=SC2086,SC2034
CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
}
@@ -904,38 +1150,38 @@ function git_checkout
cd "${BASEDIR}"
if [[ ! -d .git ]]; then
- hadoop_error "ERROR: ${BASEDIR} is not a git repo."
+ yetus_error "ERROR: ${BASEDIR} is not a git repo."
cleanup_and_exit 1
fi
if [[ ${RESETREPO} == "true" ]] ; then
${GIT} reset --hard
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git reset is failing"
+ yetus_error "ERROR: git reset is failing"
cleanup_and_exit 1
fi
# if PATCH_DIR is in BASEDIR, then we don't want
# git wiping it out.
- exemptdir=$(relative_patchdir)
+ exemptdir=$(relative_dir "${PATCH_DIR}")
if [[ $? == 1 ]]; then
${GIT} clean -xdf
else
# we do, however, want it emptied of all _files_.
# we need to leave _directories_ in case we are in
# re-exec mode (which places a directory full of stuff in it)
- hadoop_debug "Exempting ${exemptdir} from clean"
+ yetus_debug "Exempting ${exemptdir} from clean"
rm "${PATCH_DIR}/*" 2>/dev/null
${GIT} clean -xdf -e "${exemptdir}"
fi
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git clean is failing"
+ yetus_error "ERROR: git clean is failing"
cleanup_and_exit 1
fi
${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}"
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
+ yetus_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
cleanup_and_exit 1
fi
@@ -949,14 +1195,14 @@ function git_checkout
if [[ ${OFFLINE} == false ]]; then
${GIT} pull --rebase
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git pull is failing"
+ yetus_error "ERROR: git pull is failing"
cleanup_and_exit 1
fi
fi
# forcibly checkout this branch or git ref
${GIT} checkout --force "${PATCH_BRANCH}"
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
+ yetus_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
cleanup_and_exit 1
fi
@@ -965,7 +1211,7 @@ function git_checkout
if [[ ${OFFLINE} == false ]]; then
${GIT} pull --rebase
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git pull is failing"
+ yetus_error "ERROR: git pull is failing"
cleanup_and_exit 1
fi
fi
@@ -974,9 +1220,9 @@ function git_checkout
status=$(${GIT} status --porcelain)
if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then
- hadoop_error "ERROR: --dirty-workspace option not provided."
- hadoop_error "ERROR: can't run in a workspace that contains the following modifications"
- hadoop_error "${status}"
+ yetus_error "ERROR: --dirty-workspace option not provided."
+ yetus_error "ERROR: can't run in a workspace that contains the following modifications"
+ yetus_error "${status}"
cleanup_and_exit 1
fi
@@ -1005,80 +1251,8 @@ function git_checkout
echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
fi
- add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}"
+ add_footer_table "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}"
- if [[ ! -f ${BASEDIR}/pom.xml ]]; then
- hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting."
- add_jira_table -1 pre-patch "Unsupported build system."
- output_to_jira 1
- cleanup_and_exit 1
- fi
- return 0
-}
-
-## @description Confirm the source environment is compilable
-## @audience private
-## @stability stable
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_without_patch
-{
- local -r mypwd=$(pwd)
-
- big_console_header "Pre-patch ${PATCH_BRANCH} Java verification"
-
- start_clock
-
- verify_needed_test javac
-
- if [[ $? == 1 ]]; then
- echo "Compiling ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
- if [[ $? != 0 ]] ; then
- echo "${PATCH_BRANCH} compilation is broken?"
- add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need javac tests."
- fi
-
- verify_needed_test javadoc
-
- if [[ $? == 1 ]]; then
- echo "Javadoc'ing ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need javadoc tests."
- fi
-
- verify_needed_test site
-
- if [[ $? == 1 ]]; then
- echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need site tests."
- fi
-
- precheck_findbugs
-
- if [[ $? != 0 ]] ; then
- return 1
- fi
-
- add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
return 0
}
@@ -1136,7 +1310,7 @@ function determine_branch
local allbranches
local patchnamechunk
- hadoop_debug "Determine branch"
+ yetus_debug "Determine branch"
# something has already set this, so move on
if [[ -n ${PATCH_BRANCH} ]]; then
@@ -1155,12 +1329,12 @@ function determine_branch
allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g)
for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
- hadoop_debug "Determine branch: starting with ${j}"
+ yetus_debug "Determine branch: starting with ${j}"
# shellcheck disable=SC2016
patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}')
# ISSUE.branch.##.patch
- hadoop_debug "Determine branch: ISSUE.branch.##.patch"
+ yetus_debug "Determine branch: ISSUE.branch.##.patch"
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. )
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
if [[ $? == 0 ]]; then
@@ -1168,7 +1342,7 @@ function determine_branch
fi
# ISSUE-branch-##.patch
- hadoop_debug "Determine branch: ISSUE-branch-##.patch"
+ yetus_debug "Determine branch: ISSUE-branch-##.patch"
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
if [[ $? == 0 ]]; then
@@ -1176,7 +1350,7 @@ function determine_branch
fi
# ISSUE-##.patch.branch
- hadoop_debug "Determine branch: ISSUE-##.patch.branch"
+ yetus_debug "Determine branch: ISSUE-##.patch.branch"
# shellcheck disable=SC2016
PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}')
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
@@ -1185,7 +1359,7 @@ function determine_branch
fi
# ISSUE-branch.##.patch
- hadoop_debug "Determine branch: ISSUE-branch.##.patch"
+ yetus_debug "Determine branch: ISSUE-branch.##.patch"
# shellcheck disable=SC2016
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null)
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
@@ -1210,7 +1384,7 @@ function determine_issue
local patchnamechunk
local maybeissue
- hadoop_debug "Determine issue"
+ yetus_debug "Determine issue"
# we can shortcut jenkins
if [[ ${JENKINS} == true ]]; then
@@ -1241,13 +1415,13 @@ function add_test
{
local testname=$1
- hadoop_debug "Testing against ${testname}"
+ yetus_debug "Testing against ${testname}"
if [[ -z ${NEEDED_TESTS} ]]; then
- hadoop_debug "Setting tests to ${testname}"
+ yetus_debug "Setting tests to ${testname}"
NEEDED_TESTS=${testname}
elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then
- hadoop_debug "Adding ${testname}"
+ yetus_debug "Adding ${testname}"
NEEDED_TESTS="${NEEDED_TESTS} ${testname}"
fi
}
@@ -1279,44 +1453,8 @@ function determine_needed_tests
local i
for i in ${CHANGED_FILES}; do
- if [[ ${i} =~ src/main/webapp ]]; then
- hadoop_debug "tests/webapp: ${i}"
- elif [[ ${i} =~ \.sh
- || ${i} =~ \.cmd
- ]]; then
- hadoop_debug "tests/shell: ${i}"
- elif [[ ${i} =~ \.md$
- || ${i} =~ \.md\.vm$
- || ${i} =~ src/site
- || ${i} =~ src/main/docs
- ]]; then
- hadoop_debug "tests/site: ${i}"
- add_test site
- elif [[ ${i} =~ \.c$
- || ${i} =~ \.cc$
- || ${i} =~ \.h$
- || ${i} =~ \.hh$
- || ${i} =~ \.proto$
- || ${i} =~ src/test
- || ${i} =~ \.cmake$
- || ${i} =~ CMakeLists.txt
- ]]; then
- hadoop_debug "tests/units: ${i}"
- add_test javac
- add_test unit
- elif [[ ${i} =~ pom.xml$
- || ${i} =~ \.java$
- || ${i} =~ src/main
- ]]; then
- hadoop_debug "tests/javadoc+units: ${i}"
- add_test javadoc
- add_test javac
- add_test unit
- fi
- if [[ ${i} =~ \.java$ ]]; then
- add_test findbugs
- fi
+ personality_file_tests "${i}"
for plugin in ${PLUGINS}; do
if declare -f ${plugin}_filefilter >/dev/null 2>&1; then
@@ -1325,7 +1463,7 @@ function determine_needed_tests
done
done
- add_jira_footer "Optional Tests" "${NEEDED_TESTS}"
+ add_footer_table "Optional Tests" "${NEEDED_TESTS}"
}
## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the
@@ -1338,7 +1476,7 @@ function determine_needed_tests
function locate_patch
{
local notSureIfPatch=false
- hadoop_debug "locate patch"
+ yetus_debug "locate patch"
if [[ -f ${PATCH_OR_ISSUE} ]]; then
PATCH_FILE="${PATCH_OR_ISSUE}"
@@ -1349,36 +1487,64 @@ function locate_patch
else
${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
+ case $? in
+ 0)
+ ;;
+ 2)
+ yetus_error "ERROR: .wgetrc/.netrc parsing error."
+ cleanup_and_exit 1
+ ;;
+ 3)
+ yetus_error "ERROR: File IO error."
+ cleanup_and_exit 1
+ ;;
+ 4)
+ yetus_error "ERROR: URL ${PATCH_OR_ISSUE} is unreachable."
+ cleanup_and_exit 1
+ ;;
+ *)
+ # we want to try and do as much as we can in docker mode,
+ # but if the patch was passed as a file, then we may not
+ # be able to continue.
+ if [[ ${REEXECED} == true
+ && -f "${PATCH_DIR}/patch" ]]; then
+ PATCH_FILE="${PATCH_DIR}/patch"
+ else
+ yetus_error "ERROR: Unable to fetch ${PATCH_OR_ISSUE}."
+ cleanup_and_exit 1
+ fi
+ ;;
+ esac
+
+ if [[ -z "${PATCH_FILE}" ]]; then
+ if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
+ if [[ ${JENKINS} == true ]]; then
+ yetus_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
+ cleanup_and_exit 1
+ else
+ yetus_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
+ fi
+ fi
+
+ relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*')
+ PATCHURL="http://issues.apache.org${relativePatchURL}"
+ if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
+ notSureIfPatch=true
+ fi
+ patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*')
+ echo "${ISSUE} patch is being downloaded at $(date) from"
+ fi
+ fi
+ if [[ -z "${PATCH_FILE}" ]]; then
+ echo "${PATCHURL}"
+ add_footer_table "Patch URL" "${PATCHURL}"
+ ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
if [[ $? != 0 ]];then
- hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference."
+ yetus_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
cleanup_and_exit 1
fi
-
- if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
- if [[ ${JENKINS} == true ]]; then
- hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- cleanup_and_exit 1
- else
- hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- fi
- fi
-
- relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*')
- PATCHURL="http://issues.apache.org${relativePatchURL}"
- if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
- notSureIfPatch=true
- fi
- patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*')
- echo "${ISSUE} patch is being downloaded at $(date) from"
+ PATCH_FILE="${PATCH_DIR}/patch"
fi
- echo "${PATCHURL}"
- add_jira_footer "Patch URL" "${PATCHURL}"
- ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
- if [[ $? != 0 ]];then
- hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
- cleanup_and_exit 1
- fi
- PATCH_FILE="${PATCH_DIR}/patch"
fi
if [[ ! -f "${PATCH_DIR}/patch" ]]; then
@@ -1386,18 +1552,19 @@ function locate_patch
if [[ $? == 0 ]] ; then
echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}"
else
- hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
+ yetus_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
cleanup_and_exit 1
fi
fi
+
if [[ ${notSureIfPatch} == "true" ]]; then
guess_patch_file "${PATCH_DIR}/patch"
if [[ $? != 0 ]]; then
- hadoop_error "ERROR: ${PATCHURL} is not a patch file."
+ yetus_error "ERROR: ${PATCHURL} is not a patch file."
cleanup_and_exit 1
else
- hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
- add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
+ yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
+ add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
fi
fi
}
@@ -1413,15 +1580,15 @@ function guess_patch_file
local patch=$1
local fileOutput
- hadoop_debug "Trying to guess is ${patch} is a patch file."
+ yetus_debug "Trying to guess is ${patch} is a patch file."
fileOutput=$("${FILE}" "${patch}")
if [[ $fileOutput =~ \ diff\ ]]; then
- hadoop_debug "file magic says it's a diff."
+ yetus_debug "file magic says it's a diff."
return 0
fi
- fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
+ fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
if [[ $? == 0 ]]; then
- hadoop_debug "first line looks like a patch file."
+ yetus_debug "first line looks like a patch file."
return 0
fi
return 1
@@ -1442,7 +1609,7 @@ function verify_patch_file
"${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun
if [[ $? != 0 ]] ; then
echo "PATCH APPLICATION FAILED"
- add_jira_table -1 patch "The patch command could not apply the patch during dryrun."
+ add_vote_table -1 patch "The patch command could not apply the patch during dryrun."
return 1
else
return 0
@@ -1464,17 +1631,91 @@ function apply_patch_file
if [[ $? != 0 ]] ; then
echo "PATCH APPLICATION FAILED"
((RESULT = RESULT + 1))
- add_jira_table -1 patch "The patch command could not apply the patch."
+ add_vote_table -1 patch "The patch command could not apply the patch."
output_to_console 1
- output_to_jira 1
+ output_to_bugsystem 1
cleanup_and_exit 1
fi
return 0
}
+## @description copy the test-patch binary bits to a new working dir,
+## @description setting USER_PLUGIN_DIR and PERSONALITY to the new
+## @description locations.
+## @description this is used for test-patch in docker and reexec mode
+## @audience private
+## @stability evolving
+## @replaceable no
+function copytpbits
+{
+ local dockerdir
+ local dockfile
+ local person
+ # we need to copy/consolidate all the bits that might have changed
+ # that are considered part of test-patch. This *might* break
+ # things that do off-path includes, but there isn't much we can
+ # do about that, I don't think.
-## @description If this actually patches the files used for the QA process
-## @description under dev-support and its subdirectories, then
+ # if we've already copied, then don't bother doing it again
+ if [[ ${STARTDIR} == ${PATCH_DIR}/precommit ]]; then
+ hadoop_debug "Skipping copytpbits; already copied once"
+ return
+ fi
+
+ pushd "${STARTINGDIR}" >/dev/null
+ mkdir -p "${PATCH_DIR}/precommit/user-plugins"
+ mkdir -p "${PATCH_DIR}/precommit/personality"
+ mkdir -p "${PATCH_DIR}/precommit/test-patch-docker"
+
+ # copy our entire universe, preserving links, etc.
+ (cd "${BINDIR}"; tar cpf - . ) | (cd "${PATCH_DIR}/precommit"; tar xpf - )
+
+ if [[ -n "${USER_PLUGIN_DIR}"
+ && -d "${USER_PLUGIN_DIR}" ]]; then
+ cp -pr "${USER_PLUGIN_DIR}/*" \
+ "${PATCH_DIR}/precommit/user-plugins"
+ fi
+ # Set to be relative to ${PATCH_DIR}/precommit
+ USER_PLUGIN_DIR="${PATCH_DIR}/precommit/user-plugins"
+
+ if [[ -n ${PERSONALITY}
+ && -f ${PERSONALITY} ]]; then
+ cp -pr "${PERSONALITY}" "${PATCH_DIR}/precommit/personality"
+ person=$(basename "${PERSONALITY}")
+
+ # Set to be relative to ${PATCH_DIR}/precommit
+ PERSONALITY="${PATCH_DIR}/precommit/personality/${person}"
+ fi
+
+ if [[ -n ${DOCKERFILE}
+ && -f ${DOCKERFILE} ]]; then
+ dockerdir=$(dirname "${DOCKERFILE}")
+ dockfile=$(basename "${DOCKERFILE}")
+ pushd "${dockerdir}" >/dev/null
+ gitfilerev=$("${GIT}" log -n 1 --pretty=format:%h -- "${dockfile}" 2>/dev/null)
+ popd >/dev/null
+ if [[ -z ${gitfilerev} ]]; then
+ gitfilerev=$(date "+%F")
+ gitfilerev="date${gitfilerev}"
+ fi
+ (
+ echo "### TEST_PATCH_PRIVATE: dockerfile=${DOCKERFILE}"
+ echo "### TEST_PATCH_PRIVATE: gitrev=${gitfilerev}"
+ cat "${DOCKERFILE}"
+ # make sure we put some space between, just in case last
+ # line isn't an empty line or whatever
+ printf "\n\n"
+ cat "${BINDIR}/test-patch-docker/Dockerfile-endstub"
+
+ printf "\n\n"
+ ) > "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile"
+ DOCKERFILE="${PATCH_DIR}/precommit/test-patch-docker/Dockerfile"
+ fi
+
+ popd >/dev/null
+}
+
+## @description If this patches actually patches test-patch.sh, then
## @description run with the patched version for the test.
## @audience private
## @stability evolving
@@ -1483,54 +1724,563 @@ function apply_patch_file
function check_reexec
{
local commentfile=${PATCH_DIR}/tp.${RANDOM}
+ local tpdir
+ local copy=false
+ local testdir
+ local person
if [[ ${REEXECED} == true ]]; then
big_console_header "Re-exec mode detected. Continuing."
return
fi
- if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch
- && ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then
+ for testdir in "${BINDIR}" \
+ "${PERSONALITY}" \
+ "${USER_PLUGIN_DIR}" \
+ "${DOCKERFILE}"; do
+ tpdir=$(relative_dir "${testdir}")
+ if [[ $? == 0
+ && ${CHANGED_FILES} =~ ${tpdir} ]]; then
+ copy=true
+ fi
+ done
+
+ if [[ ${copy} == true ]]; then
+ big_console_header "precommit patch detected"
+
+ if [[ ${RESETREPO} == false ]]; then
+ ((RESULT = RESULT + 1))
+ yetus_debug "can't destructively change the working directory. run with '--resetrepo' please. :("
+ add_vote_table -1 precommit "Couldn't test precommit changes because we aren't configured to destructively change the working directory."
+ else
+
+ apply_patch_file
+
+ if [[ ${JENKINS} == true ]]; then
+ rm "${commentfile}" 2>/dev/null
+ echo "(!) A patch to the testing environment has been detected. " > "${commentfile}"
+ echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}"
+ echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}"
+ write_comment "${commentfile}"
+ rm "${commentfile}"
+ fi
+ fi
+ fi
+
+ if [[ ${DOCKERSUPPORT} == false
+ && ${copy} == false ]]; then
return
fi
- big_console_header "dev-support patch detected"
+ if [[ ${DOCKERSUPPORT} == true
+ && ${copy} == false ]]; then
+ big_console_header "Re-execing under Docker"
- if [[ ${RESETREPO} == false ]]; then
- ((RESULT = RESULT + 1))
- hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :("
- add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory."
- return
fi
- printf "\n\nRe-executing against patched versions to test.\n\n"
+ # copy our universe
+ copytpbits
- apply_patch_file
+ if [[ ${DOCKERSUPPORT} == true ]]; then
+ # if we are doing docker, then we re-exec, but underneath the
+ # container
- if [[ ${JENKINS} == true ]]; then
+ client=$(docker version | grep 'Client version' | cut -f2 -d: | tr -d ' ')
+ server=$(docker version | grep 'Server version' | cut -f2 -d: | tr -d ' ')
- rm "${commentfile}" 2>/dev/null
+ dockerversion="Client=${client} Server=${server}"
- echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}"
- echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}"
- echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}"
+ TESTPATCHMODE="${USER_PARAMS[*]}"
+ if [[ -n "${BUILD_URL}" ]]; then
+ TESTPATCHMODE="--build-url=${BUILD_URL} ${TESTPATCHMODE}"
+ fi
+ TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}"
+ TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}"
+ TESTPATCHMODE="--personality=\'${PERSONALITY}\' ${TESTPATCHMODE}"
+ TESTPATCHMODE="--plugins=\'${USER_PLUGIN_DIR}\' ${TESTPATCHMODE}"
+ TESTPATCHMODE=" ${TESTPATCHMODE}"
+ export TESTPATCHMODE
- write_to_jira "${commentfile}"
- rm "${commentfile}"
+ patchdir=$(relative_dir "${PATCH_DIR}")
+
+ cd "${BASEDIR}"
+ #shellcheck disable=SC2093
+ exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \
+ --dockerversion="${dockerversion}" \
+ --java-home="${JAVA_HOME}" \
+ --patch-dir="${patchdir}" \
+ --project="${PROJECT_NAME}"
+
+ else
+
+ # if we aren't doing docker, then just call ourselves
+ # but from the new path with the new flags
+ #shellcheck disable=SC2093
+ cd "${PATCH_DIR}/precommit/"
+ exec "${PATCH_DIR}/precommit/test-patch.sh" \
+ "${USER_PARAMS[@]}" \
+ --reexec \
+ --basedir="${BASEDIR}" \
+ --branch="${PATCH_BRANCH}" \
+ --patch-dir="${PATCH_DIR}" \
+ --tpglobaltimer="${GLOBALTIMER}" \
+ --tpreexectimer="${TIMER}" \
+ --personality="${PERSONALITY}" \
+ --plugins="${USER_PLUGIN_DIR}"
+ fi
+}
+
+## @description Reset the test results
+## @audience public
+## @stability evolving
+## @replaceable no
+function modules_reset
+{
+ MODULE_STATUS=()
+ MODULE_STATUS_TIMER=()
+ MODULE_STATUS_MSG=()
+ MODULE_STATUS_LOG=()
+}
+
+## @description Utility to print standard module errors
+## @audience public
+## @stability evolving
+## @replaceable no
+## @param repostatus
+## @param testtype
+## @param mvncmdline
+function modules_messages
+{
+ local repostatus=$1
+ local testtype=$2
+ local summarymode=$3
+ shift 2
+ local modindex=0
+ local repo
+ local goodtime=0
+ local failure=false
+ local oldtimer
+ local statusjdk
+ local multijdkmode=false
+
+ if [[ ${repostatus} == branch ]]; then
+ repo=${PATCH_BRANCH}
+ else
+ repo="the patch"
fi
- cd "${CWD}"
- mkdir -p "${PATCH_DIR}/dev-support-test"
- cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test"
- cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test"
+ verify_multijdk_test "${testtype}"
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
- big_console_header "exec'ing test-patch.sh now..."
+ oldtimer=${TIMER}
- exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
- --reexec \
- --branch="${PATCH_BRANCH}" \
- --patch-dir="${PATCH_DIR}" \
- "${USER_PARAMS[@]}"
+ if [[ ${summarymode} == true
+ && ${ALLOWSUMMARIES} == true ]]; then
+
+ until [[ ${modindex} -eq ${#MODULE[@]} ]]; do
+
+ if [[ ${multijdkmode} == true ]]; then
+ statusjdk=${MODULE_STATUS_JDK[${modindex}]}
+ fi
+
+ if [[ "${MODULE_STATUS[${modindex}]}" == '+1' ]]; then
+ ((goodtime=goodtime + ${MODULE_STATUS_TIMER[${modindex}]}))
+ else
+ failure=true
+ start_clock
+ echo ""
+ echo "${MODULE_STATUS_MSG[${modindex}]}"
+ echo ""
+ offset_clock "${MODULE_STATUS_TIMER[${modindex}]}"
+ add_vote_table "${MODULE_STATUS[${modindex}]}" "${testtype}" "${MODULE_STATUS_MSG[${modindex}]}"
+ if [[ ${MODULE_STATUS[${modindex}]} == -1
+ && -n "${MODULE_STATUS_LOG[${modindex}]}" ]]; then
+ add_footer_table "${testtype}" "@@BASE@@/${MODULE_STATUS_LOG[${modindex}]}"
+ fi
+ fi
+ ((modindex=modindex+1))
+ done
+
+ if [[ ${failure} == false ]]; then
+ start_clock
+ offset_clock "${goodtime}"
+ add_vote_table +1 "${testtype}" "${repo} passed${statusjdk}"
+ fi
+ else
+ until [[ ${modindex} -eq ${#MODULE[@]} ]]; do
+ start_clock
+ echo ""
+ echo "${MODULE_STATUS_MSG[${modindex}]}"
+ echo ""
+ offset_clock "${MODULE_STATUS_TIMER[${modindex}]}"
+ add_vote_table "${MODULE_STATUS[${modindex}]}" "${testtype}" "${MODULE_STATUS_MSG[${modindex}]}"
+ if [[ ${MODULE_STATUS[${modindex}]} == -1
+ && -n "${MODULE_STATUS_LOG[${modindex}]}" ]]; then
+ add_footer_table "${testtype}" "@@BASE@@/${MODULE_STATUS_LOG[${modindex}]}"
+ fi
+ ((modindex=modindex+1))
+ done
+ fi
+ TIMER=${oldtimer}
+}
+
+## @description Add a test result
+## @audience public
+## @stability evolving
+## @replaceable no
+## @param module
+## @param runtime
+function module_status
+{
+ local index=$1
+ local value=$2
+ local log=$3
+ shift 3
+
+ local jdk
+
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+
+ if [[ -n ${index}
+ && ${index} =~ ^[0-9]+$ ]]; then
+ MODULE_STATUS[${index}]="${value}"
+ MODULE_STATUS_LOG[${index}]="${log}"
+ MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}"
+ MODULE_STATUS_MSG[${index}]="${*}"
+ else
+ yetus_error "ASSERT: module_status given bad index: ${index}"
+ local frame=0
+ while caller $frame; do
+ ((frame++));
+ done
+ echo "$*"
+ exit 1
+ fi
+}
+
+## @description run the maven tests for the queued modules
+## @audience public
+## @stability evolving
+## @replaceable no
+## @param repostatus
+## @param testtype
+## @param mvncmdline
+function modules_workers
+{
+ local repostatus=$1
+ local testtype=$2
+ shift 2
+ local modindex=0
+ local fn
+ local savestart=${TIMER}
+ local savestop
+ local repo
+ local modulesuffix
+ local jdk=""
+ local jdkindex=0
+ local statusjdk
+
+ if [[ ${repostatus} == branch ]]; then
+ repo=${PATCH_BRANCH}
+ else
+ repo="the patch"
+ fi
+
+ modules_reset
+
+ verify_multijdk_test "${testtype}"
+ if [[ $? == 1 ]]; then
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+ statusjdk=" with JDK v${jdk}"
+ jdk="-jdk${jdk}"
+ jdk=${jdk// /}
+ yetus_debug "Starting MultiJDK mode${statusjdk} on ${testtype}"
+ fi
+
+ until [[ ${modindex} -eq ${#MODULE[@]} ]]; do
+ start_clock
+
+ fn=$(module_file_fragment "${MODULE[${modindex}]}")
+ fn="${fn}${jdk}"
+ modulesuffix=$(basename "${MODULE[${modindex}]}")
+ pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null
+
+ if [[ ${modulesuffix} == . ]]; then
+ modulesuffix="root"
+ fi
+
+ if [[ $? != 0 ]]; then
+ echo "${BASEDIR}/${MODULE[${modindex}]} no longer exists. Skipping."
+ ((modindex=modindex+1))
+ continue
+ fi
+
+ case ${BUILDTOOL} in
+ maven)
+ #shellcheck disable=SC2086
+ echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \
+ ${MVN} "${MAVEN_ARGS[@]}" \
+ "${@//@@@MODULEFN@@@/${fn}}" \
+ ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} -Ptest-patch
+ ;;
+ ant)
+ #shellcheck disable=SC2086
+ echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \
+ "${ANT}" "${ANT_ARGS[@]}" \
+ ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} \
+ "${@//@@@MODULEFN@@@/${fn}}"
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ return 1
+ ;;
+ esac
+
+ if [[ $? == 0 ]] ; then
+ module_status \
+ ${modindex} \
+ +1 \
+ "${repostatus}-${testtype}-${fn}.txt" \
+ "${modulesuffix} in ${repo} passed${statusjdk}."
+ else
+ module_status \
+ ${modindex} \
+ -1 \
+ "${repostatus}-${testtype}-${fn}.txt" \
+ "${modulesuffix} in ${repo} failed${statusjdk}."
+ ((result = result + 1))
+ fi
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${modindex}]=${savestop}
+ # shellcheck disable=SC2086
+ echo "Elapsed: $(clock_display ${savestop})"
+ popd >/dev/null
+ ((modindex=modindex+1))
+ done
+
+ TIMER=${savestart}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Reset the queue for tests
+## @audience public
+## @stability evolving
+## @replaceable no
+function clear_personality_queue
+{
+ yetus_debug "Personality: clear queue"
+ MODCOUNT=0
+ MODULE=()
+}
+
+## @description Build the queue for tests
+## @audience public
+## @stability evolving
+## @replaceable no
+## @param module
+## @param profiles/flags/etc
+function personality_enqueue_module
+{
+ yetus_debug "Personality: enqueue $*"
+ local module=$1
+ shift
+
+ MODULE[${MODCOUNT}]=${module}
+ MODULEEXTRAPARAM[${MODCOUNT}]=${*}
+ ((MODCOUNT=MODCOUNT+1))
+}
+
+## @description Confirm compilation pre-patch
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function precheck_javac
+{
+ local result=0
+ local -r savejavahome=${JAVA_HOME}
+ local multijdkmode=false
+ local jdkindex=0
+
+ big_console_header "Pre-patch ${PATCH_BRANCH} javac compilation"
+
+ verify_needed_test javac
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need javac tests."
+ return 0
+ fi
+
+ verify_multijdk_test javac
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
+
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ fi
+
+ personality_modules branch javac
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers branch javac clean compile
+ ;;
+ ant)
+ modules_workers branch javac
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ return 1
+ ;;
+ esac
+
+ ((result=result + $?))
+ modules_messages branch javac true
+
+ done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Confirm Javadoc pre-patch
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function precheck_javadoc
+{
+ local result=0
+ local -r savejavahome=${JAVA_HOME}
+ local multijdkmode=false
+ local jdkindex=0
+
+ big_console_header "Pre-patch ${PATCH_BRANCH} Javadoc verification"
+
+ verify_needed_test javadoc
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need javadoc tests."
+ return 0
+ fi
+
+ verify_multijdk_test javadoc
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
+
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ fi
+
+ personality_modules branch javadoc
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers branch javadoc clean javadoc:javadoc
+ ;;
+ ant)
+ modules_workers branch javadoc clean javadoc
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ return 1
+ ;;
+ esac
+
+ ((result=result + $?))
+ modules_messages branch javadoc true
+
+ done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Confirm site pre-patch
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function precheck_site
+{
+ local result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ big_console_header "Pre-patch ${PATCH_BRANCH} site verification"
+
+ verify_needed_test site
+ if [[ $? == 0 ]];then
+ echo "Patch does not appear to need site tests."
+ return 0
+ fi
+
+ personality_modules branch site
+ modules_workers branch site clean site site:stage
+ result=$?
+ modules_messages branch site true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Confirm the source environment pre-patch
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function precheck_without_patch
+{
+ local result=0
+
+ precheck_mvninstall
+
+ if [[ $? -gt 0 ]]; then
+ ((result = result +1 ))
+ fi
+
+ precheck_javac
+
+ if [[ $? -gt 0 ]]; then
+ ((result = result +1 ))
+ fi
+
+ precheck_javadoc
+
+ if [[ $? -gt 0 ]]; then
+ ((result = result +1 ))
+ fi
+
+ precheck_site
+
+ if [[ $? -gt 0 ]]; then
+ ((result = result +1 ))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ return 0
}
## @description Check the current directory for @author tags
@@ -1542,25 +2292,27 @@ function check_reexec
function check_author
{
local authorTags
+ local -r appname=$(basename "${BASH_SOURCE-$0}")
big_console_header "Checking there are no @author tags in the patch."
- start_clock
-
- if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then
- add_jira_table 0 @author "Skipping @author checks as test-patch has been patched."
+ if [[ ${CHANGED_FILES} =~ ${appname} ]]; then
+ echo "Skipping @author checks as ${appname} has been patched."
+ add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched."
return 0
fi
+ start_clock
+
authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch")
echo "There appear to be ${authorTags} @author tags in the patch."
if [[ ${authorTags} != 0 ]] ; then
- add_jira_table -1 @author \
- "The patch appears to contain ${authorTags} @author tags which the Hadoop" \
+ add_vote_table -1 @author \
+ "The patch appears to contain ${authorTags} @author tags which the" \
" community has agreed to not allow in code contributions."
return 1
fi
- add_jira_table +1 @author "The patch does not contain any @author tags."
+ add_vote_table +1 @author "The patch does not contain any @author tags."
return 0
}
@@ -1575,14 +2327,15 @@ function check_modified_unittests
local testReferences=0
local i
+ big_console_header "Checking there are new or changed tests in the patch."
+
verify_needed_test unit
if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need new or modified tests."
return 0
fi
- big_console_header "Checking there are new or changed tests in the patch."
-
start_clock
for i in ${CHANGED_FILES}; do
@@ -1593,29 +2346,184 @@ function check_modified_unittests
echo "There appear to be ${testReferences} test file(s) referenced in the patch."
if [[ ${testReferences} == 0 ]] ; then
- add_jira_table -1 "tests included" \
+ add_vote_table -1 "test4tests" \
"The patch doesn't appear to include any new or modified tests. " \
"Please justify why no new tests are needed for this patch." \
"Also please list what manual steps were performed to verify this patch."
return 1
fi
- add_jira_table +1 "tests included" \
+ add_vote_table +1 "test4tests" \
"The patch appears to include ${testReferences} new or modified test files."
return 0
}
-## @description Helper for check_javadoc
+## @description Helper for check_patch_javac
## @audience private
## @stability evolving
## @replaceable no
## @return 0 on success
## @return 1 on failure
-function count_javadoc_warns
+function count_javac_probs
{
local warningfile=$1
+ local val1
+ local val2
- #shellcheck disable=SC2016,SC2046
- return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}')
+ case ${BUILDTOOL} in
+ maven)
+ #shellcheck disable=SC2016,SC2046
+ ${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}"
+ ;;
+ ant)
+ #shellcheck disable=SC2016
+ val1=$(${GREP} -E "\[javac\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ #shellcheck disable=SC2016
+ val2=$(${GREP} -E "\[javac\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ echo $((val1+val2))
+ ;;
+ esac
+}
+
+## @description Count and compare the number of javac warnings pre- and post- patch
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function check_patch_javac
+{
+ local i
+ local result=0
+ local fn
+ local -r savejavahome=${JAVA_HOME}
+ local multijdkmode=false
+ local jdk=""
+ local jdkindex=0
+ local statusjdk
+ declare -i numbranch=0
+ declare -i numpatch=0
+
+ big_console_header "Determining number of patched javac errors"
+
+ verify_needed_test javac
+
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need javac tests."
+ return 0
+ fi
+
+ verify_multijdk_test javac
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
+
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+ yetus_debug "Using ${JAVA_HOME} to run this set of tests"
+ statusjdk=" with JDK v${jdk}"
+ jdk="-jdk${jdk}"
+ jdk=${jdk// /}
+ fi
+
+ personality_modules patch javac
+
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers patch javac clean compile
+ ;;
+ ant)
+ modules_workers patch javac
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ return 1
+ ;;
+ esac
+
+ i=0
+ until [[ ${i} -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+
+ fn=$(module_file_fragment "${MODULE[${i}]}")
+ fn="${fn}${jdk}"
+ module_suffix=$(basename "${MODULE[${i}]}")
+ if [[ ${module_suffix} == \. ]]; then
+ module_suffix=root
+ fi
+
+ # if it was a new module, this won't exist.
+ if [[ -f "${PATCH_DIR}/branch-javac-${fn}.txt" ]]; then
+ ${GREP} '\[WARNING\]' "${PATCH_DIR}/branch-javac-${fn}.txt" \
+ > "${PATCH_DIR}/branch-javac-${fn}-warning.txt"
+ else
+ touch "${PATCH_DIR}/branch-javac-${fn}.txt" \
+ "${PATCH_DIR}/branch-javac-${fn}-warning.txt"
+ fi
+
+ ${GREP} '\[WARNING\]' "${PATCH_DIR}/patch-javac-${fn}.txt" \
+ > "${PATCH_DIR}/patch-javac-${fn}-warning.txt"
+
+ numbranch=$(count_javac_probs "${PATCH_DIR}/branch-javac-${fn}-warning.txt")
+ numpatch=$(count_javac_probs "${PATCH_DIR}/patch-javac-${fn}-warning.txt")
+
+ if [[ -n ${numbranch}
+ && -n ${numpatch}
+ && ${numpatch} -gt ${numbranch} ]]; then
+
+ ${DIFF} -u "${PATCH_DIR}/branch-javac-${fn}-warning.txt" \
+ "${PATCH_DIR}/patch-javac-${fn}-warning.txt" \
+ > "${PATCH_DIR}/javac-${fn}-diff.txt"
+
+ module_status ${i} -1 "javac-${fn}-diff.txt" \
+ "Patched ${module_suffix} generated "\
+ "$((numpatch-numbranch)) additional warning messages${statusjdk}." \
+
+ ((result=result+1))
+ fi
+ ((i=i+1))
+ done
+
+ modules_messages patch javac true
+ done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Helper for check_patch_javadoc
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function count_javadoc_probs
+{
+ local warningfile=$1
+ local val1
+ local val2
+
+ case ${BUILDTOOL} in
+ maven)
+ #shellcheck disable=SC2016,SC2046
+ ${GREP} -E "^[0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}'
+ ;;
+ ant)
+ #shellcheck disable=SC2016
+ val1=$(${GREP} -E "\[javadoc\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ #shellcheck disable=SC2016
+ val2=$(${GREP} -E "\[javadoc\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ echo $((val1+val2))
+ ;;
+ esac
}
## @description Count and compare the number of JavaDoc warnings pre- and post- patch
@@ -1624,53 +2532,106 @@ function count_javadoc_warns
## @replaceable no
## @return 0 on success
## @return 1 on failure
-function check_javadoc
+function check_patch_javadoc
{
- local numBranchJavadocWarnings
- local numPatchJavadocWarnings
-
- verify_needed_test javadoc
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need javadoc checks."
- return 0
- fi
+ local i
+ local result=0
+ local fn
+ local -r savejavahome=${JAVA_HOME}
+ local multijdkmode=false
+ local jdk=""
+ local jdkindex=0
+ local statusjdk
+ declare -i numbranch=0
+ declare -i numpatch=0
big_console_header "Determining number of patched javadoc warnings"
- start_clock
-
- if [[ -d hadoop-project ]]; then
- (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
+ verify_needed_test javadoc
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need javadoc tests."
+ return 0
fi
- if [[ -d hadoop-common-project/hadoop-annotations ]]; then
- (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
+
+ verify_multijdk_test javadoc
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
fi
- echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
- count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
- numBranchJavadocWarnings=$?
- count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
- numPatchJavadocWarnings=$?
- echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch."
- if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then
- if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then
-
- ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt"
- ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
- ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \
- "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \
- > "${PATCH_DIR}/diffJavadocWarnings.txt"
- rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
-
- add_jira_table -1 javadoc "The applied patch generated "\
- "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \
- " additional warning messages."
- add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt"
- return 1
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+ yetus_debug "Using ${JAVA_HOME} to run this set of tests"
+ statusjdk=" with JDK v${jdk}"
+ jdk="-jdk${jdk}"
+ jdk=${jdk// /}
fi
+
+ personality_modules patch javadoc
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers patch javadoc clean javadoc:javadoc
+ ;;
+ ant)
+ modules_workers patch javadoc clean javadoc
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ return 1
+ ;;
+ esac
+
+ i=0
+ until [[ ${i} -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+
+ fn=$(module_file_fragment "${MODULE[${i}]}")
+ fn="${fn}${jdk}"
+ numbranch=$(count_javadoc_probs "${PATCH_DIR}/branch-javadoc-${fn}.txt")
+ numpatch=$(count_javadoc_probs "${PATCH_DIR}/patch-javadoc-${fn}.txt")
+
+ if [[ -n ${numbranch}
+ && -n ${numpatch}
+ && ${numpatch} -gt ${numbranch} ]] ; then
+
+ if [[ -f "${PATCH_DIR}/branch-javadoc-${fn}.txt" ]]; then
+ ${GREP} -i warning "${PATCH_DIR}/branch-javadoc-${fn}.txt" \
+ > "${PATCH_DIR}/branch-javadoc-${fn}-filtered.txt"
+ else
+ touch "${PATCH_DIR}/branch-javadoc-${fn}.txt" \
+ "${PATCH_DIR}/branch-javadoc-${fn}-filtered.txt"
+ fi
+
+ ${GREP} -i warning "${PATCH_DIR}/patch-javadoc-${fn}.txt" \
+ > "${PATCH_DIR}/patch-javadoc-${fn}-filtered.txt"
+
+ ${DIFF} -u "${PATCH_DIR}/branch-javadoc-${fn}-filtered.txt" \
+ "${PATCH_DIR}/patch-javadoc-${fn}-filtered.txt" \
+ > "${PATCH_DIR}/javadoc-${fn}-diff.txt"
+ rm -f "${PATCH_DIR}/branch-javadoc-${fn}-filtered.txt" \
+ "${PATCH_DIR}/patch-javadoc-${fn}-filtered.txt"
+
+ module_status ${i} -1 "javadoc-${fn}-diff.txt" \
+ "Patched ${MODULE[${i}]} generated "\
+ "$((numpatch-numbranch)) additional warning messages${statusjdk}."
+
+ ((result=result+1))
+ fi
+ ((i=i+1))
+ done
+
+ modules_messages patch javadoc true
+ done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
fi
- add_jira_table +1 javadoc "There were no new javadoc warning messages."
return 0
}
@@ -1682,144 +2643,27 @@ function check_javadoc
## @return 1 on failure
function check_site
{
- local -r mypwd=$(pwd)
+ local result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ big_console_header "Determining number of patched site errors"
verify_needed_test site
-
if [[ $? == 0 ]]; then
- echo "This patch does not appear to need site checks."
+ echo "Patch does not appear to need site tests."
return 0
fi
- big_console_header "Determining if patched site still builds"
-
- start_clock
-
- echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Site compilation is broken"
- add_jira_table -1 site "Site compilation is broken."
- add_jira_footer site "@@BASE@@/patchSiteWarnings.txt"
+ personality_modules patch site
+ modules_workers patch site clean site site:stage -Dmaven.javadoc.skip=true
+ result=$?
+ modules_messages patch site true
+ if [[ ${result} != 0 ]]; then
return 1
fi
- add_jira_table +1 site "Site still builds."
- return 0
-}
-
-## @description Helper for check_javac
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javac_warns
-{
- local warningfile=$1
- #shellcheck disable=SC2016,SC2046
- return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}")
-}
-
-## @description Count and compare the number of javac warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_javac
-{
- local branchJavacWarnings
- local patchJavacWarnings
-
- verify_needed_test javac
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need javac checks."
- return 0
- fi
-
- big_console_header "Determining number of patched javac warnings."
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
- if [[ $? != 0 ]] ; then
- add_jira_table -1 javac "The patch appears to cause the build to fail."
- return 2
- fi
- ### Compare ${PATCH_BRANCH} and patch javac warning numbers
- if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then
- ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
- ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
-
- count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
- branchJavacWarnings=$?
- count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
- patchJavacWarnings=$?
-
- echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch."
- if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then
- if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then
-
- ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \
- "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \
- > "${PATCH_DIR}/diffJavacWarnings.txt"
-
- add_jira_table -1 javac "The applied patch generated "\
- "$((patchJavacWarnings-branchJavacWarnings))" \
- " additional warning messages."
-
- add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt"
-
- return 1
- fi
- fi
- fi
-
- add_jira_table +1 javac "There were no new javac warning messages."
- return 0
-}
-
-## @description Verify all files have an Apache License
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_apachelicense
-{
- big_console_header "Determining number of patched release audit warnings."
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
- #shellcheck disable=SC2038
- find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
-
- ### Compare ${PATCH_BRANCH} and patch release audit warning numbers
- if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then
- patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt")
- echo ""
- echo ""
- echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch."
- if [[ ${patchReleaseAuditWarnings} != "" ]] ; then
- if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then
- add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings."
-
- ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \
- > "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
- echo "Lines that start with ????? in the release audit "\
- "report indicate files that do not have an Apache license header." \
- >> "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
- add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt"
-
- return 1
- fi
- fi
- fi
- add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings."
return 0
}
@@ -1829,284 +2673,69 @@ function check_apachelicense
## @replaceable no
## @return 0 on success
## @return 1 on failure
-function check_mvn_install
+function precheck_mvninstall
{
- local retval
+ local result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ big_console_header "Verifying mvn install works"
verify_needed_test javadoc
retval=$?
verify_needed_test javac
((retval = retval + $? ))
-
if [[ ${retval} == 0 ]]; then
echo "This patch does not appear to need mvn install checks."
return 0
fi
- big_console_header "Installing all of the jars"
+ personality_modules branch mvninstall
+ modules_workers branch mvninstall -fae clean install -Dmaven.javadoc.skip=true
+ result=$?
+ modules_messages branch mvninstall true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
- start_clock
- echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
+## @description Verify mvn install works
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function check_mvninstall
+{
+ local result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ big_console_header "Verifying mvn install still works"
+
+ verify_needed_test javadoc
retval=$?
- if [[ ${retval} != 0 ]]; then
- add_jira_table -1 install "The patch causes mvn install to fail."
- else
- add_jira_table +1 install "mvn install still works."
- fi
- return ${retval}
-}
-## @description are the needed bits for findbugs present?
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 findbugs will work for our use
-## @return 1 findbugs is missing some component
-function findbugs_is_installed
-{
- if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
- printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
- add_jira_table -1 findbugs "Findbugs is not installed."
- return 1
- fi
- return 0
-}
-
-## @description Run the maven findbugs plugin and record found issues in a bug database
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function findbugs_mvnrunner
-{
- local name=$1
- local logfile=$2
- local warnings_file=$3
-
- echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
- "-D${PROJECT_NAME}PatchProcess" < /dev/null
- if [[ $? != 0 ]]; then
- return 1
- fi
- cp target/findbugsXml.xml "${warnings_file}.xml"
-
- "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
- "${warnings_file}.xml" "${warnings_file}.xml"
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
- "${warnings_file}.html"
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- return 0
-}
-
-## @description Track pre-existing findbugs warnings
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_findbugs
-{
- local -r mypwd=$(pwd)
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
- local findbugs_version
- local rc=0
- local module_findbugs_warnings
- local findbugs_warnings=0
-
- verify_needed_test findbugs
-
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need findbugs tests."
+ verify_needed_test javac
+ ((retval = retval + $? ))
+ if [[ ${retval} == 0 ]]; then
+ echo "This patch does not appear to need mvn install checks."
return 0
fi
- echo "findbugs baseline for ${mypwd}"
-
- findbugs_is_installed
- if [[ $? != 0 ]]; then
+ personality_modules patch mvninstall
+ modules_workers patch mvninstall install -Dmaven.javadoc.skip=true
+ result=$?
+ modules_messages patch mvninstall true
+ if [[ ${result} != 0 ]]; then
return 1
fi
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running findbugs in ${module}"
- module_suffix=$(basename "${module}")
- findbugs_mvnrunner "${PATCH_BRANCH}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
- (( rc = rc + $? ))
-
- if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
- #shellcheck disable=SC2016
- module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
- "${PATCH_BRANCH}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
- | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
-
- if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
- add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
- fi
- fi
- popd >/dev/null
- done
-
- #shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
-
- if [[ ${rc} -ne 0 ]]; then
- echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
- add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
- return 1
- fi
-
- if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
- ${findbugs_warnings} -gt 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
-
- return 0
-}
-
-## @description Verify patch does not trigger any findbugs warnings
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_findbugs
-{
- local rc=0
- local module
- local modules=${CHANGED_MODULES}
- local module_suffix
- local combined_xml
- local newBugs
- local new_findbugs_warnings
- local new_findbugs_fixed_warnings
- local findbugs_warnings=0
- local findbugs_fixed_warnings=0
- local line
- local firstpart
- local secondpart
- local findbugs_version
-
- verify_needed_test findbugs
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "Determining number of patched Findbugs warnings."
-
- start_clock
-
- findbugs_is_installed
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running findbugs in ${module}"
- module_suffix=$(basename "${module}")
-
- findbugs_mvnrunner patch \
- "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch findbugs compilation is broken."
- add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
- continue
- fi
-
- combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
- newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
- "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
- -output "${combined_xml}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- #shellcheck disable=SC2016
- new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
- "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
- #shellcheck disable=SC2016
- new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
- "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
- findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
- findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
-
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
- "${newBugs}.html"
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
- populate_test_table FindBugs "module:${module_suffix}"
- while read line; do
- firstpart=$(echo "${line}" | cut -f2 -d:)
- secondpart=$(echo "${line}" | cut -f9- -d' ')
- add_jira_test_table "" "${firstpart}:${secondpart}"
- done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
-
- add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
- fi
-
- popd >/dev/null
- done
-
- #shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
-
- if [[ ${findbugs_warnings} -gt 0 ]] ; then
- add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
-
- if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
- else
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
- fi
return 0
}
@@ -2118,7 +2747,11 @@ function check_findbugs
## @return 1 on failure
function check_mvn_eclipse
{
- big_console_header "Running mvn eclipse:eclipse."
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ big_console_header "Verifying mvn eclipse:eclipse still works"
verify_needed_test javac
if [[ $? == 0 ]]; then
@@ -2126,14 +2759,13 @@ function check_mvn_eclipse
return 0
fi
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
+ personality_modules patch eclipse
+ modules_workers patch eclipse eclipse:eclipse
+ result=$?
+ modules_messages patch eclipse true
+ if [[ ${result} != 0 ]]; then
return 1
fi
- add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse."
return 0
}
@@ -2152,10 +2784,10 @@ function populate_test_table
for i in "$@"; do
if [[ -z "${first}" ]]; then
- add_jira_test_table "${reason}" "${i}"
+ add_test_table "${reason}" "${i}"
first="${reason}"
else
- add_jira_test_table " " "${i}"
+ add_test_table " " "${i}"
fi
done
}
@@ -2168,6 +2800,20 @@ function populate_test_table
## @return 1 on failure
function check_unittests
{
+ local i
+ local failed_tests=""
+ local test_timeouts=""
+ local test_logfile
+ local module_test_timeouts=""
+ local result=0
+ local -r savejavahome=${JAVA_HOME}
+ local multijdkmode=false
+ local jdk=""
+ local jdkindex=0
+ local statusjdk
+
+ big_console_header "Running unit tests"
+
verify_needed_test unit
if [[ $? == 0 ]]; then
@@ -2175,128 +2821,92 @@ function check_unittests
return 0
fi
- big_console_header "Running unit tests"
+ verify_multijdk_test unit
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
- start_clock
-
- local failed_tests=""
- local modules=${CHANGED_MODULES}
- local building_common=0
- local hdfs_modules
- local ordered_modules=""
- local failed_test_builds=""
- local test_timeouts=""
- local test_logfile
- local test_build_result
- local module_test_timeouts=""
- local result
- local totalresult=0
- local module_prefix
-
- #
- # If we are building hadoop-hdfs-project, we must build the native component
- # of hadoop-common-project first. In order to accomplish this, we move the
- # hadoop-hdfs subprojects to the end of the list so that common will come
- # first.
- #
- # Of course, we may not be building hadoop-common at all-- in this case, we
- # explicitly insert a mvn compile -Pnative of common, to ensure that the
- # native libraries show up where we need them.
- #
-
- for module in ${modules}; do
- if [[ ${module} == hadoop-hdfs-project* ]]; then
- hdfs_modules="${hdfs_modules} ${module}"
- elif [[ ${module} == hadoop-common-project* ]]; then
- ordered_modules="${ordered_modules} ${module}"
- building_common=1
- else
- ordered_modules="${ordered_modules} ${module}"
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+ statusjdk="JDK v${jdk} "
+ jdk="-jdk${jdk}"
+ jdk=${jdk// /}
fi
- done
- if [[ -n "${hdfs_modules}" ]]; then
- ordered_modules="${ordered_modules} ${hdfs_modules}"
- if [[ ${building_common} -eq 0 ]]; then
- echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
- echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
- if [[ $? != 0 ]]; then
- add_jira_table -1 "native" "Failed to build the native portion " \
- "of hadoop-common prior to running the unit tests in ${ordered_modules}"
+ personality_modules patch unit
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers patch unit clean install -fae
+ ;;
+ ant)
+ modules_workers patch unit
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
return 1
- else
- add_jira_table +1 "native" "Pre-build of native portion"
+ ;;
+ esac
+ ((result=result+$?))
+
+ modules_messages patch unit false
+ if [[ ${result} == 0 ]]; then
+ continue
+ fi
+
+ i=0
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ module=${MODULE[${i}]}
+ fn=$(module_file_fragment "${module}")
+ fn="${fn}${jdk}"
+ test_logfile="${PATCH_DIR}/patch-unit-${fn}.txt"
+
+ # shellcheck disable=2016
+ module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${test_logfile}")
+ if [[ -n "${module_test_timeouts}" ]] ; then
+ test_timeouts="${test_timeouts} ${module_test_timeouts}"
+ ((result=result+1))
fi
+
+ pushd "${MODULE[${i}]}" >/dev/null
+ #shellcheck disable=SC2026,SC2038,SC2016
+ module_failed_tests=$(find . -name 'TEST*.xml'\
+ | xargs "${GREP}" -l -E "/dev/null
+
+ if [[ -n "${module_failed_tests}" ]] ; then
+ failed_tests="${failed_tests} ${module_failed_tests}"
+ ((result=result+1))
+ fi
+
+ ((i=i+1))
+ done
+
+ if [[ -n "${failed_tests}" ]] ; then
+ # shellcheck disable=SC2086
+ populate_test_table "${statusjdk}Failed unit tests" ${failed_tests}
+ failed_tests=""
fi
- fi
-
- for module in ${ordered_modules}; do
- result=0
- start_clock
- pushd "${module}" >/dev/null
- module_suffix=$(basename "${module}")
- module_prefix=$(echo "${module}" | cut -f2 -d- )
-
- test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
- echo " Running tests in ${module_suffix}"
- echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} -D${PROJECT_NAME}PatchProcess
- test_build_result=$?
-
- add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"
-
- # shellcheck disable=2016
- module_test_timeouts=$(${AWK} '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' "${test_logfile}")
- if [[ -n "${module_test_timeouts}" ]] ; then
- test_timeouts="${test_timeouts} ${module_test_timeouts}"
- result=1
+ if [[ -n "${test_timeouts}" ]] ; then
+ # shellcheck disable=SC2086
+ populate_test_table "${statusjdk}Timed out tests" ${test_timeouts}
+ test_timeouts=""
fi
- #shellcheck disable=SC2026,SC2038,SC2016
- module_failed_tests=$(find . -name 'TEST*.xml'\
- | xargs "${GREP}" -l -E "/dev/null
-
- if [[ $result == 1 ]]; then
- add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}."
- else
- add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}."
- fi
-
- ((totalresult = totalresult + result))
done
-
- if [[ -n "${failed_tests}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Failed unit tests" ${failed_tests}
- fi
- if [[ -n "${test_timeouts}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Timed out tests" ${test_timeouts}
- fi
- if [[ -n "${failed_test_builds}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Failed build" ${failed_test_builds}
- fi
+ JAVA_HOME=${savejavahome}
if [[ ${JENKINS} == true ]]; then
- add_jira_footer "Test Results" "${BUILD_URL}testReport/"
+ add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
fi
- if [[ ${totalresult} -gt 0 ]]; then
+ if [[ ${result} -gt 0 ]]; then
return 1
- else
- return 0
fi
+ return 0
}
## @description Print out the finished details on the console
@@ -2310,7 +2920,7 @@ function output_to_console
{
local result=$1
shift
- local i
+ local i=0
local ourstring
local vote
local subs
@@ -2359,25 +2969,24 @@ function output_to_console
rm "${spcfx}"
fi
- seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}")
+ seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}")
if [[ ${seccoladj} -lt 10 ]]; then
seccoladj=10
fi
seccoladj=$((seccoladj + 2 ))
i=0
- until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
- printf "%s\n" "${JIRA_HEADER[${i}]}"
+ until [[ $i -eq ${#TP_HEADER[@]} ]]; do
+ printf "%s\n" "${TP_HEADER[${i}]}"
((i=i+1))
done
printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
echo "============================================================================"
i=0
- until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
- ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ')
+ until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
+ ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
vote=$(echo "${ourstring}" | cut -f2 -d\|)
- vote=$(colorstripper "${vote}")
subs=$(echo "${ourstring}" | cut -f3 -d\|)
ela=$(echo "${ourstring}" | cut -f4 -d\|)
comment=$(echo "${ourstring}" | cut -f5 -d\|)
@@ -2396,12 +3005,12 @@ function output_to_console
rm "${commentfile2}" "${commentfile1}" 2>/dev/null
done
- if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
- seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}")
+ if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
+ seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}")
printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
i=0
- until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
- ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ')
+ until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
+ ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
vote=$(echo "${ourstring}" | cut -f2 -d\|)
subs=$(echo "${ourstring}" | cut -f3 -d\|)
printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
@@ -2413,87 +3022,21 @@ function output_to_console
echo "============================================================================"
i=0
- until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
+ until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
+ comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
printf "%s\n" "${comment}"
((i=i+1))
done
}
-## @description Print out the finished details to the JIRA issue
+## @description Write the final output to the selected bug system
## @audience private
## @stability evolving
## @replaceable no
-## @param runresult
-function output_to_jira
+function output_to_bugsystem
{
- local result=$1
- local i
- local commentfile=${PATCH_DIR}/commentfile
- local comment
-
- rm "${commentfile}" 2>/dev/null
-
- if [[ ${JENKINS} != "true" ]] ; then
- return 0
- fi
-
- big_console_header "Adding comment to JIRA"
-
- add_jira_footer "Console output" "${BUILD_URL}console"
-
- if [[ ${result} == 0 ]]; then
- add_jira_header "(/) *{color:green}+1 overall{color}*"
- else
- add_jira_header "(x) *{color:red}-1 overall{color}*"
- fi
-
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- i=0
- until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
- printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}"
-
- i=0
- until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
- printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
-
-
- if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Reason || Tests ||" >> "${commentfile}"
- i=0
- until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
- printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
- fi
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Subsystem || Report/Notes ||" >> "${commentfile}"
- i=0
- until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g")
- printf "%s\n" "${comment}" >> "${commentfile}"
- ((i=i+1))
- done
-
- printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
-
- write_to_jira "${commentfile}"
+ "${BUGSYSTEM}_finalreport" "${@}"
}
## @description Clean the filesystem as appropriate and then exit
@@ -2511,9 +3054,9 @@ function cleanup_and_exit
# there is no need to move it since we assume that
# Jenkins or whatever already knows where it is at
# since it told us to put it there!
- relative_patchdir >/dev/null
+ relative_dir "${PATCH_DIR}" >/dev/null
if [[ $? == 1 ]]; then
- hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
+ yetus_debug "mv ${PATCH_DIR} ${BASEDIR}"
mv "${PATCH_DIR}" "${BASEDIR}"
fi
fi
@@ -2532,17 +3075,16 @@ function postcheckout
local routine
local plugin
- for routine in find_java_home verify_patch_file
- do
+ for routine in find_java_home verify_patch_file; do
verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
+ yetus_debug "Running ${routine}"
${routine}
(( RESULT = RESULT + $? ))
if [[ ${RESULT} != 0 ]] ; then
output_to_console 1
- output_to_jira 1
+ output_to_bugsystem 1
cleanup_and_exit 1
fi
done
@@ -2552,14 +3094,14 @@ function postcheckout
if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postcheckout"
+ yetus_debug "Running ${plugin}_postcheckout"
#shellcheck disable=SC2086
${plugin}_postcheckout
(( RESULT = RESULT + $? ))
if [[ ${RESULT} != 0 ]] ; then
output_to_console 1
- output_to_jira 1
+ output_to_bugsystem 1
cleanup_and_exit 1
fi
fi
@@ -2580,7 +3122,7 @@ function preapply
do
verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
+ yetus_debug "Running ${routine}"
${routine}
(( RESULT = RESULT + $? ))
@@ -2591,7 +3133,7 @@ function preapply
if declare -f ${plugin}_preapply >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_preapply"
+ yetus_debug "Running ${plugin}_preapply"
#shellcheck disable=SC2086
${plugin}_preapply
@@ -2610,32 +3152,30 @@ function postapply
local plugin
local retval
- compute_gitdiff "${GITDIFFLINES}"
+ compute_gitdiff
- check_javac
+ check_patch_javac
retval=$?
if [[ ${retval} -gt 1 ]] ; then
output_to_console 1
- output_to_jira 1
+ output_to_bugsystem 1
cleanup_and_exit 1
fi
((RESULT = RESULT + retval))
- for routine in check_javadoc check_apachelicense check_site
+ for routine in check_patch_javadoc check_site
do
verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
- $routine
-
+ yetus_debug "Running ${routine}"
+ ${routine}
(( RESULT = RESULT + $? ))
-
done
for plugin in ${PLUGINS}; do
verify_patchdir_still_exists
if declare -f ${plugin}_postapply >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postapply"
+ yetus_debug "Running ${plugin}_postapply"
#shellcheck disable=SC2086
${plugin}_postapply
(( RESULT = RESULT + $? ))
@@ -2652,24 +3192,19 @@ function postinstall
local routine
local plugin
- for routine in check_mvn_eclipse check_findbugs
- do
- verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
- ${routine}
- (( RESULT = RESULT + $? ))
- done
+ verify_patchdir_still_exists
+ check_mvn_eclipse
+ (( RESULT = RESULT + $? ))
for plugin in ${PLUGINS}; do
verify_patchdir_still_exists
if declare -f ${plugin}_postinstall >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postinstall"
+ yetus_debug "Running ${plugin}_postinstall"
#shellcheck disable=SC2086
${plugin}_postinstall
(( RESULT = RESULT + $? ))
fi
done
-
}
## @description Driver to execute _tests routines
@@ -2692,7 +3227,7 @@ function runtests
for plugin in ${PLUGINS}; do
verify_patchdir_still_exists
if declare -f ${plugin}_tests >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_tests"
+ yetus_debug "Running ${plugin}_tests"
#shellcheck disable=SC2086
${plugin}_tests
(( RESULT = RESULT + $? ))
@@ -2717,13 +3252,49 @@ function importplugins
fi
if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
- hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
+ yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
fi
for i in "${files[@]}"; do
- hadoop_debug "Importing ${i}"
- . "${i}"
+ if [[ -f ${i} ]]; then
+ yetus_debug "Importing ${i}"
+ . "${i}"
+ fi
+ done
+
+ if [[ -z ${PERSONALITY}
+ && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" ]]; then
+ PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
+ fi
+
+ if [[ -n ${PERSONALITY} ]]; then
+ if [[ ! -f ${PERSONALITY} ]]; then
+ if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh" ]]; then
+ PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
+ else
+ yetus_debug "Can't find ${PERSONALITY} to import."
+ return
+ fi
+ fi
+ yetus_debug "Importing ${PERSONALITY}"
+ . "${PERSONALITY}"
+ fi
+}
+
+## @description Let plugins also get a copy of the arguments
+## @audience private
+## @stability evolving
+## @replaceable no
+function parse_args_plugins
+{
+ for plugin in ${PLUGINS} ${BUGSYSTEMS}; do
+ if declare -f ${plugin}_parse_args >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_parse_args"
+ #shellcheck disable=SC2086
+ ${plugin}_parse_args "$@"
+ (( RESULT = RESULT + $? ))
+ fi
done
}
@@ -2736,6 +3307,15 @@ function add_plugin
PLUGINS="${PLUGINS} $1"
}
+## @description Register test-patch.d plugins
+## @audience public
+## @stability stable
+## @replaceable no
+function add_bugsystem
+{
+ BUGSYSTEMS="${BUGSYSTEMS} $1"
+}
+
###############################################################################
###############################################################################
###############################################################################
@@ -2748,12 +3328,12 @@ parse_args "$@"
importplugins
+parse_args_plugins "$@"
+
+finish_docker_stats
+
locate_patch
-find_changed_files
-
-determine_needed_tests
-
# from here on out, we'll be in ${BASEDIR} for cwd
# routines need to pushd/popd if they change.
git_checkout
@@ -2764,28 +3344,40 @@ if [[ ${JENKINS} == "true" ]] ; then
fi
fi
+find_changed_files
+
check_reexec
+determine_needed_tests
+
postcheckout
+fullyqualifyjdks
+
+prepopulate_footer
+
find_changed_modules
preapply
apply_patch_file
+# we find changed modules again
+# in case the patch adds or removes a module
+# this also means that test suites need to be
+# aware that there might not be a 'before'
+find_changed_modules
+
postapply
-check_mvn_install
+check_mvninstall
postinstall
runtests
-close_jira_footer
-
-close_jira_table
+finish_vote_table
output_to_console ${RESULT}
-output_to_jira ${RESULT}
+output_to_bugsystem ${RESULT}
cleanup_and_exit ${RESULT}
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HADOOP-12111.txt b/hadoop-common-project/hadoop-common/CHANGES-HADOOP-12111.txt
index 0ec7a0f75d3..cb2f8887d47 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HADOOP-12111.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HADOOP-12111.txt
@@ -1 +1,3 @@
Breakdown of HADOOP-12111 sub-tasks:
+
+ HADOOP-12113. update test-patch branch to latest code (aw)