2809 lines
79 KiB
Bash
Executable File
2809 lines
79 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
### BUILD_URL is set by Hudson if it is run by patch process
|
|
|
|
this="${BASH_SOURCE-$0}"
|
|
BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
|
|
CWD=$(pwd)
|
|
USER_PARAMS=("$@")
|
|
GLOBALTIMER=$(date +"%s")
|
|
|
|
## @description Setup the default global variables
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
function setup_defaults
|
|
{
|
|
if [[ -z "${MAVEN_HOME:-}" ]]; then
|
|
MVN=mvn
|
|
else
|
|
MVN=${MAVEN_HOME}/bin/mvn
|
|
fi
|
|
# This parameter needs to be kept as an array
|
|
MAVEN_ARGS=()
|
|
|
|
PROJECT_NAME=hadoop
|
|
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
|
|
JENKINS=false
|
|
BASEDIR=$(pwd)
|
|
RELOCATE_PATCH_DIR=false
|
|
|
|
USER_PLUGIN_DIR=""
|
|
LOAD_SYSTEM_PLUGINS=true
|
|
|
|
FINDBUGS_HOME=${FINDBUGS_HOME:-}
|
|
FINDBUGS_WARNINGS_FAIL_PRECHECK=false
|
|
ECLIPSE_HOME=${ECLIPSE_HOME:-}
|
|
BUILD_NATIVE=${BUILD_NATIVE:-true}
|
|
PATCH_BRANCH=""
|
|
PATCH_BRANCH_DEFAULT="trunk"
|
|
CHANGED_MODULES=""
|
|
USER_MODULE_LIST=""
|
|
OFFLINE=false
|
|
CHANGED_FILES=""
|
|
REEXECED=false
|
|
RESETREPO=false
|
|
ISSUE=""
|
|
ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
|
|
TIMER=$(date +"%s")
|
|
PATCHURL=""
|
|
|
|
OSTYPE=$(uname -s)
|
|
|
|
# Solaris needs POSIX, not SVID
|
|
case ${OSTYPE} in
|
|
SunOS)
|
|
PS=${PS:-ps}
|
|
AWK=${AWK:-/usr/xpg4/bin/awk}
|
|
SED=${SED:-/usr/xpg4/bin/sed}
|
|
WGET=${WGET:-wget}
|
|
GIT=${GIT:-git}
|
|
EGREP=${EGREP:-/usr/xpg4/bin/egrep}
|
|
GREP=${GREP:-/usr/xpg4/bin/grep}
|
|
PATCH=${PATCH:-patch}
|
|
DIFF=${DIFF:-/usr/gnu/bin/diff}
|
|
JIRACLI=${JIRA:-jira}
|
|
FILE=${FILE:-file}
|
|
;;
|
|
*)
|
|
PS=${PS:-ps}
|
|
AWK=${AWK:-awk}
|
|
SED=${SED:-sed}
|
|
WGET=${WGET:-wget}
|
|
GIT=${GIT:-git}
|
|
EGREP=${EGREP:-egrep}
|
|
GREP=${GREP:-grep}
|
|
PATCH=${PATCH:-patch}
|
|
DIFF=${DIFF:-diff}
|
|
JIRACLI=${JIRA:-jira}
|
|
FILE=${FILE:-file}
|
|
;;
|
|
esac
|
|
|
|
declare -a JIRA_COMMENT_TABLE
|
|
declare -a JIRA_FOOTER_TABLE
|
|
declare -a JIRA_HEADER
|
|
declare -a JIRA_TEST_TABLE
|
|
|
|
JFC=0
|
|
JTC=0
|
|
JTT=0
|
|
RESULT=0
|
|
}
|
|
|
|
## @description Print a message to stderr
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param string
|
|
function hadoop_error
|
|
{
|
|
echo "$*" 1>&2
|
|
}
|
|
|
|
## @description Print a message to stderr if --debug is turned on
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param string
|
|
function hadoop_debug
|
|
{
|
|
if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
|
|
echo "[$(date) DEBUG]: $*" 1>&2
|
|
fi
|
|
}
|
|
|
|
## @description Activate the local timer
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
function start_clock
|
|
{
|
|
hadoop_debug "Start clock"
|
|
TIMER=$(date +"%s")
|
|
}
|
|
|
|
## @description Print the elapsed time in seconds since the start of the local timer
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
function stop_clock
|
|
{
|
|
local -r stoptime=$(date +"%s")
|
|
local -r elapsed=$((stoptime-TIMER))
|
|
hadoop_debug "Stop clock"
|
|
|
|
echo ${elapsed}
|
|
}
|
|
|
|
## @description Print the elapsed time in seconds since the start of the global timer
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
function stop_global_clock
|
|
{
|
|
local -r stoptime=$(date +"%s")
|
|
local -r elapsed=$((stoptime-GLOBALTIMER))
|
|
hadoop_debug "Stop global clock"
|
|
|
|
echo ${elapsed}
|
|
}
|
|
|
|
## @description Add time to the local timer
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param seconds
|
|
function offset_clock
|
|
{
|
|
((TIMER=TIMER-$1))
|
|
}
|
|
|
|
## @description Add to the header of the display
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param string
|
|
function add_jira_header
|
|
{
|
|
JIRA_HEADER[${JHC}]="| $* |"
|
|
JHC=$(( JHC+1 ))
|
|
}
|
|
|
|
## @description Add to the output table. If the first parameter is a number
|
|
## @description that is the vote for that column and calculates the elapsed time
|
|
## @description based upon the last start_clock(). If it the string null, then it is
|
|
## @description a special entry that signifies extra
|
|
## @description content for the final column. The second parameter is the reporting
|
|
## @description subsystem (or test) that is providing the vote. The second parameter
|
|
## @description is always required. The third parameter is any extra verbage that goes
|
|
## @description with that subsystem.
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param +1/0/-1/null
|
|
## @param subsystem
|
|
## @param string
|
|
## @return Elapsed time display
|
|
function add_jira_table
|
|
{
|
|
local value=$1
|
|
local subsystem=$2
|
|
shift 2
|
|
|
|
local color
|
|
local calctime=0
|
|
|
|
local -r elapsed=$(stop_clock)
|
|
|
|
if [[ ${elapsed} -lt 0 ]]; then
|
|
calctime="N/A"
|
|
else
|
|
printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
|
|
fi
|
|
|
|
echo ""
|
|
echo "Elapsed time: ${calctime}"
|
|
echo ""
|
|
|
|
case ${value} in
|
|
1|+1)
|
|
value="+1"
|
|
color="green"
|
|
;;
|
|
-1)
|
|
color="red"
|
|
;;
|
|
0)
|
|
color="blue"
|
|
;;
|
|
null)
|
|
;;
|
|
esac
|
|
|
|
if [[ -z ${color} ]]; then
|
|
JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |"
|
|
JTC=$(( JTC+1 ))
|
|
else
|
|
JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |"
|
|
JTC=$(( JTC+1 ))
|
|
fi
|
|
}
|
|
|
|
## @description Put the final environment information at the bottom
|
|
## @description of the footer table
|
|
## @stability stable
|
|
## @audience private
|
|
## @replaceable yes
|
|
function close_jira_footer
|
|
{
|
|
# shellcheck disable=SC2016
|
|
local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \")
|
|
local -r unamea=$(uname -a)
|
|
|
|
add_jira_footer "Java" "${javaversion}"
|
|
add_jira_footer "uname" "${unamea}"
|
|
}
|
|
|
|
## @description Put the final elapsed time at the bottom of the table.
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
function close_jira_table
|
|
{
|
|
|
|
local -r elapsed=$(stop_global_clock)
|
|
|
|
if [[ ${elapsed} -lt 0 ]]; then
|
|
calctime="N/A"
|
|
else
|
|
printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
|
|
fi
|
|
|
|
echo ""
|
|
echo "Total Elapsed time: ${calctime}"
|
|
echo ""
|
|
|
|
|
|
JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |"
|
|
JTC=$(( JTC+1 ))
|
|
}
|
|
|
|
## @description Add to the footer of the display. @@BASE@@ will get replaced with the
|
|
## @description correct location for the local filesystem in dev mode or the URL for
|
|
## @description Jenkins mode.
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param subsystem
|
|
## @param string
|
|
function add_jira_footer
|
|
{
|
|
local subsystem=$1
|
|
shift 1
|
|
|
|
JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |"
|
|
JFC=$(( JFC+1 ))
|
|
}
|
|
|
|
## @description Special table just for unit test failures
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param failurereason
|
|
## @param testlist
|
|
function add_jira_test_table
|
|
{
|
|
local failure=$1
|
|
shift 1
|
|
|
|
JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |"
|
|
JTT=$(( JTT+1 ))
|
|
}
|
|
|
|
## @description Large display for the user console
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param string
|
|
## @return large chunk of text
|
|
function big_console_header
|
|
{
|
|
local text="$*"
|
|
local spacing=$(( (75+${#text}) /2 ))
|
|
printf "\n\n"
|
|
echo "============================================================================"
|
|
echo "============================================================================"
|
|
printf "%*s\n" ${spacing} "${text}"
|
|
echo "============================================================================"
|
|
echo "============================================================================"
|
|
printf "\n\n"
|
|
}
|
|
|
|
## @description Remove {color} tags from a string
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param string
|
|
## @return string
|
|
function colorstripper
|
|
{
|
|
local string=$1
|
|
shift 1
|
|
|
|
local green=""
|
|
local white=""
|
|
local red=""
|
|
local blue=""
|
|
|
|
echo "${string}" | \
|
|
${SED} -e "s,{color:red},${red},g" \
|
|
-e "s,{color:green},${green},g" \
|
|
-e "s,{color:blue},${blue},g" \
|
|
-e "s,{color},${white},g"
|
|
}
|
|
|
|
## @description Find the largest size of a column of an array
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return size
|
|
function findlargest
|
|
{
|
|
local column=$1
|
|
shift
|
|
local a=("$@")
|
|
local sizeofa=${#a[@]}
|
|
local i=0
|
|
|
|
until [[ ${i} -gt ${sizeofa} ]]; do
|
|
# shellcheck disable=SC2086
|
|
string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| )
|
|
if [[ ${#string} -gt $maxlen ]]; then
|
|
maxlen=${#string}
|
|
fi
|
|
i=$((i+1))
|
|
done
|
|
echo "${maxlen}"
|
|
}
|
|
|
|
## @description Verify that ${JAVA_HOME} is defined
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @return 1 - no JAVA_HOME
|
|
## @return 0 - JAVA_HOME defined
|
|
function find_java_home
|
|
{
|
|
start_clock
|
|
if [[ -z ${JAVA_HOME:-} ]]; then
|
|
case $(uname -s) in
|
|
Darwin)
|
|
if [[ -z "${JAVA_HOME}" ]]; then
|
|
if [[ -x /usr/libexec/java_home ]]; then
|
|
JAVA_HOME="$(/usr/libexec/java_home)"
|
|
export JAVA_HOME
|
|
else
|
|
export JAVA_HOME=/Library/Java/Home
|
|
fi
|
|
fi
|
|
;;
|
|
*)
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [[ -z ${JAVA_HOME:-} ]]; then
|
|
echo "JAVA_HOME is not defined."
|
|
add_jira_table -1 pre-patch "JAVA_HOME is not defined."
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
## @description Write the contents of a file to jenkins
|
|
## @params filename
|
|
## @stability stable
|
|
## @audience public
|
|
## @returns ${JIRACLI} exit code
|
|
function write_to_jira
|
|
{
|
|
local -r commentfile=${1}
|
|
shift
|
|
|
|
local retval
|
|
|
|
if [[ ${OFFLINE} == false
|
|
&& ${JENKINS} == true ]]; then
|
|
export USER=hudson
|
|
# shellcheck disable=SC2086
|
|
${JIRACLI} --comment "$(cat ${commentfile})" \
|
|
-s https://issues.apache.org/jira \
|
|
-a addcomment -u hadoopqa \
|
|
-p "${JIRA_PASSWD}" \
|
|
--issue "${ISSUE}"
|
|
retval=$?
|
|
${JIRACLI} -s https://issues.apache.org/jira \
|
|
-a logout -u hadoopqa \
|
|
-p "${JIRA_PASSWD}"
|
|
fi
|
|
return ${retval}
|
|
}
|
|
|
|
## @description Verify that the patch directory is still in working order
|
|
## @description since bad actors on some systems wipe it out. If not,
|
|
## @description recreate it and then exit
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable yes
|
|
## @returns may exit on failure
|
|
function verify_patchdir_still_exists
|
|
{
|
|
local -r commentfile=/tmp/testpatch.$$.${RANDOM}
|
|
local extra=""
|
|
|
|
if [[ ! -d ${PATCH_DIR} ]]; then
|
|
rm "${commentfile}" 2>/dev/null
|
|
|
|
echo "(!) The patch artifact directory has been removed! " > "${commentfile}"
|
|
echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}"
|
|
echo
|
|
cat ${commentfile}
|
|
echo
|
|
if [[ ${JENKINS} == true ]]; then
|
|
if [[ -n ${NODE_NAME} ]]; then
|
|
extra=" (node ${NODE_NAME})"
|
|
fi
|
|
echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}"
|
|
|
|
write_to_jira ${commentfile}
|
|
fi
|
|
|
|
rm "${commentfile}"
|
|
cleanup_and_exit ${RESULT}
|
|
fi
|
|
}
|
|
|
|
## @description generate a list of all files and line numbers that
|
|
## @description that were added/changed in the source repo
|
|
## @audience private
|
|
## @stability stable
|
|
## @params filename
|
|
## @replaceable no
|
|
function compute_gitdiff
|
|
{
|
|
local outfile=$1
|
|
local file
|
|
local line
|
|
local startline
|
|
local counter
|
|
local numlines
|
|
local actual
|
|
|
|
pushd "${BASEDIR}" >/dev/null
|
|
while read line; do
|
|
if [[ ${line} =~ ^\+\+\+ ]]; then
|
|
file="./"$(echo "${line}" | cut -f2- -d/)
|
|
continue
|
|
elif [[ ${line} =~ ^@@ ]]; then
|
|
startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + )
|
|
numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, )
|
|
# if this is empty, then just this line
|
|
# if it is 0, then no lines were added and this part of the patch
|
|
# is strictly a delete
|
|
if [[ ${numlines} == 0 ]]; then
|
|
continue
|
|
elif [[ -z ${numlines} ]]; then
|
|
numlines=1
|
|
fi
|
|
counter=0
|
|
until [[ ${counter} -gt ${numlines} ]]; do
|
|
((actual=counter+startline))
|
|
echo "${file}:${actual}:" >> "${outfile}"
|
|
((counter=counter+1))
|
|
done
|
|
fi
|
|
done < <("${GIT}" diff --unified=0 --no-color)
|
|
popd >/dev/null
|
|
}
|
|
|
|
## @description Print the command to be executing to the screen. Then
|
|
## @description run the command, sending stdout and stderr to the given filename
|
|
## @description This will also ensure that any directories in ${BASEDIR} have
|
|
## @description the exec bit set as a pre-exec step.
|
|
## @audience public
|
|
## @stability stable
|
|
## @param filename
|
|
## @param command
|
|
## @param [..]
|
|
## @replaceable no
|
|
## @returns $?
|
|
function echo_and_redirect
|
|
{
|
|
local logfile=$1
|
|
shift
|
|
|
|
verify_patchdir_still_exists
|
|
|
|
find "${BASEDIR}" -type d -exec chmod +x {} \;
|
|
echo "${*} > ${logfile} 2>&1"
|
|
"${@}" > "${logfile}" 2>&1
|
|
}
|
|
|
|
## @description is PATCH_DIR relative to BASEDIR?
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable yes
|
|
## @returns 1 - no, PATCH_DIR
|
|
## @returns 0 - yes, PATCH_DIR - BASEDIR
|
|
function relative_patchdir
|
|
{
|
|
local p=${PATCH_DIR#${BASEDIR}}
|
|
|
|
if [[ ${#p} -eq ${#PATCH_DIR} ]]; then
|
|
echo ${p}
|
|
return 1
|
|
fi
|
|
p=${p#/}
|
|
echo ${p}
|
|
return 0
|
|
}
|
|
|
|
|
|
## @description Print the usage information
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
function hadoop_usage
|
|
{
|
|
local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]')
|
|
|
|
echo "Usage: test-patch.sh [options] patch-file | issue-number | http"
|
|
echo
|
|
echo "Where:"
|
|
echo " patch-file is a local patch file containing the changes to test"
|
|
echo " issue-number is a 'Patch Available' JIRA defect number (e.g. '${up}-9902') to test"
|
|
echo " http is an HTTP address to download the patch file"
|
|
echo
|
|
echo "Options:"
|
|
echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
|
|
echo "--branch=<ref> Forcibly set the branch"
|
|
echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')"
|
|
echo "--build-native=<bool> If true, then build native components (default 'true')"
|
|
echo "--contrib-guide=<url> URL to point new users towards project conventions. (default Hadoop's wiki)"
|
|
echo "--debug If set, then output some extra stuff to stderr"
|
|
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
|
|
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
|
|
echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
|
|
echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
|
|
echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
|
|
echo "--offline Avoid connecting to the Internet"
|
|
echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')"
|
|
echo "--plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)"
|
|
echo "--project=<name> The short name for project currently using test-patch (default 'hadoop')"
|
|
echo "--resetrepo Forcibly clean the repo"
|
|
echo "--run-tests Run all relevant tests below the base directory"
|
|
echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d"
|
|
echo "--testlist=<list> Specify which subsystem tests to use (comma delimited)"
|
|
echo "--test-parallel=<bool> Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)"
|
|
echo "--test-threads=<int> Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
|
|
|
|
echo "Shell binary overrides:"
|
|
echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
|
|
echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')"
|
|
echo "--file-cmd=<cmd> The 'file' command to use (default 'file')"
|
|
echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
|
|
echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
|
|
echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
|
|
echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
|
|
echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
|
|
echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')"
|
|
|
|
echo
|
|
echo "Jenkins-only options:"
|
|
echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
|
|
echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
|
|
echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
|
|
echo "--jira-password=<pw> The password for the 'jira' command"
|
|
echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
|
|
echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
|
|
}
|
|
|
|
## @description Interpret the command line parameters
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @params $@
|
|
## @return May exit on failure
|
|
function parse_args
|
|
{
|
|
local i
|
|
local j
|
|
|
|
for i in "$@"; do
|
|
case ${i} in
|
|
--awk-cmd=*)
|
|
AWK=${i#*=}
|
|
;;
|
|
--basedir=*)
|
|
BASEDIR=${i#*=}
|
|
;;
|
|
--branch=*)
|
|
PATCH_BRANCH=${i#*=}
|
|
;;
|
|
--branch-default=*)
|
|
PATCH_BRANCH_DEFAULT=${i#*=}
|
|
;;
|
|
--build-native=*)
|
|
BUILD_NATIVE=${i#*=}
|
|
;;
|
|
--contrib-guide=*)
|
|
HOW_TO_CONTRIBUTE=${i#*=}
|
|
;;
|
|
--debug)
|
|
HADOOP_SHELL_SCRIPT_DEBUG=true
|
|
;;
|
|
--diff-cmd=*)
|
|
DIFF=${i#*=}
|
|
;;
|
|
--dirty-workspace)
|
|
DIRTY_WORKSPACE=true
|
|
;;
|
|
--eclipse-home=*)
|
|
ECLIPSE_HOME=${i#*=}
|
|
;;
|
|
--file-cmd=*)
|
|
FILE=${i#*=}
|
|
;;
|
|
--findbugs-home=*)
|
|
FINDBUGS_HOME=${i#*=}
|
|
;;
|
|
--findbugs-strict-precheck)
|
|
FINDBUGS_WARNINGS_FAIL_PRECHECK=true
|
|
;;
|
|
--git-cmd=*)
|
|
GIT=${i#*=}
|
|
;;
|
|
--grep-cmd=*)
|
|
GREP=${i#*=}
|
|
;;
|
|
--help|-help|-h|help|--h|--\?|-\?|\?)
|
|
hadoop_usage
|
|
exit 0
|
|
;;
|
|
--issue-re=*)
|
|
ISSUE_RE=${i#*=}
|
|
;;
|
|
--java-home=*)
|
|
JAVA_HOME=${i#*=}
|
|
;;
|
|
--jenkins)
|
|
JENKINS=true
|
|
TEST_PARALLEL=${TEST_PARALLEL:-true}
|
|
;;
|
|
--jira-cmd=*)
|
|
JIRACLI=${i#*=}
|
|
;;
|
|
--jira-password=*)
|
|
JIRA_PASSWD=${i#*=}
|
|
;;
|
|
--modulelist=*)
|
|
USER_MODULE_LIST=${i#*=}
|
|
USER_MODULE_LIST=${USER_MODULE_LIST//,/ }
|
|
hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}"
|
|
;;
|
|
--mvn-cmd=*)
|
|
MVN=${i#*=}
|
|
;;
|
|
--mv-patch-dir)
|
|
RELOCATE_PATCH_DIR=true;
|
|
;;
|
|
--offline)
|
|
OFFLINE=true
|
|
;;
|
|
--patch-cmd=*)
|
|
PATCH=${i#*=}
|
|
;;
|
|
--patch-dir=*)
|
|
USER_PATCH_DIR=${i#*=}
|
|
;;
|
|
--plugins=*)
|
|
USER_PLUGIN_DIR=${i#*=}
|
|
;;
|
|
--project=*)
|
|
PROJECT_NAME=${i#*=}
|
|
;;
|
|
--ps-cmd=*)
|
|
PS=${i#*=}
|
|
;;
|
|
--reexec)
|
|
REEXECED=true
|
|
start_clock
|
|
add_jira_table 0 reexec "dev-support patch detected."
|
|
;;
|
|
--resetrepo)
|
|
RESETREPO=true
|
|
;;
|
|
--run-tests)
|
|
RUN_TESTS=true
|
|
;;
|
|
--skip-system-plugins)
|
|
LOAD_SYSTEM_PLUGINS=false
|
|
;;
|
|
--testlist=*)
|
|
testlist=${i#*=}
|
|
testlist=${testlist//,/ }
|
|
for j in ${testlist}; do
|
|
hadoop_debug "Manually adding patch test subsystem ${j}"
|
|
add_test "${j}"
|
|
done
|
|
;;
|
|
--test-parallel=*)
|
|
TEST_PARALLEL=${i#*=}
|
|
;;
|
|
--test-threads=*)
|
|
TEST_THREADS=${i#*=}
|
|
;;
|
|
--wget-cmd=*)
|
|
WGET=${i#*=}
|
|
;;
|
|
*)
|
|
PATCH_OR_ISSUE=${i}
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# if we requested offline, pass that to mvn
|
|
if [[ ${OFFLINE} == "true" ]] ; then
|
|
MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
|
|
fi
|
|
|
|
# we need absolute dir for ${BASEDIR}
|
|
cd "${CWD}"
|
|
BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
|
|
|
|
if [[ ${BUILD_NATIVE} == "true" ]] ; then
|
|
NATIVE_PROFILE=-Pnative
|
|
REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
|
|
fi
|
|
if [[ -z "${PATCH_OR_ISSUE}" ]]; then
|
|
hadoop_usage
|
|
exit 1
|
|
fi
|
|
if [[ ${JENKINS} == "true" ]] ; then
|
|
echo "Running in Jenkins mode"
|
|
ISSUE=${PATCH_OR_ISSUE}
|
|
RESETREPO=true
|
|
# shellcheck disable=SC2034
|
|
ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}"
|
|
else
|
|
if [[ ${RESETREPO} == "true" ]] ; then
|
|
echo "Running in destructive (--resetrepo) developer mode"
|
|
else
|
|
echo "Running in developer mode"
|
|
fi
|
|
JENKINS=false
|
|
fi
|
|
|
|
if [[ -n ${USER_PATCH_DIR} ]]; then
|
|
PATCH_DIR="${USER_PATCH_DIR}"
|
|
else
|
|
PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$
|
|
fi
|
|
|
|
cd "${CWD}"
|
|
if [[ ! -d ${PATCH_DIR} ]]; then
|
|
mkdir -p "${PATCH_DIR}"
|
|
if [[ $? == 0 ]] ; then
|
|
echo "${PATCH_DIR} has been created"
|
|
else
|
|
echo "Unable to create ${PATCH_DIR}"
|
|
cleanup_and_exit 1
|
|
fi
|
|
fi
|
|
|
|
# we need absolute dir for PATCH_DIR
|
|
PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
|
|
|
|
GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt
|
|
|
|
if [[ ${TEST_PARALLEL} == "true" ]] ; then
|
|
PARALLEL_TESTS_PROFILE=-Pparallel-tests
|
|
if [[ -n ${TEST_THREADS:-} ]]; then
|
|
TESTS_THREAD_COUNT="-DtestsThreadCount=$TEST_THREADS"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
## @description Locate the pom.xml file for a given directory
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @return directory containing the pom.xml
|
|
function find_pom_dir
|
|
{
|
|
local dir
|
|
|
|
dir=$(dirname "$1")
|
|
|
|
hadoop_debug "Find pom dir for: ${dir}"
|
|
|
|
while builtin true; do
|
|
if [[ -f "${dir}/pom.xml" ]];then
|
|
echo "${dir}"
|
|
hadoop_debug "Found: ${dir}"
|
|
return
|
|
else
|
|
dir=$(dirname "${dir}")
|
|
fi
|
|
done
|
|
}
|
|
|
|
## @description List of files that ${PATCH_DIR}/patch modifies
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @return None; sets ${CHANGED_FILES}
|
|
function find_changed_files
|
|
{
|
|
# get a list of all of the files that have been changed,
|
|
# except for /dev/null (which would be present for new files).
|
|
# Additionally, remove any a/ b/ patterns at the front
|
|
# of the patch filenames and any revision info at the end
|
|
# shellcheck disable=SC2016
|
|
CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \
|
|
| ${SED} \
|
|
-e 's,^....,,' \
|
|
-e 's,^[ab]/,,' \
|
|
| ${GREP} -v /dev/null \
|
|
| ${AWK} '{print $1}' \
|
|
| sort -u)
|
|
}
|
|
|
|
## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @return None; sets ${CHANGED_MODULES}
|
|
function find_changed_modules
|
|
{
|
|
# Come up with a list of changed files into ${TMP}
|
|
local pomdirs
|
|
local module
|
|
local pommods
|
|
|
|
# Now find all the modules that were changed
|
|
for file in ${CHANGED_FILES}; do
|
|
#shellcheck disable=SC2086
|
|
pomdirs="${pomdirs} $(find_pom_dir ${file})"
|
|
done
|
|
|
|
# Filter out modules without code
|
|
for module in ${pomdirs}; do
|
|
${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null
|
|
if [[ "$?" != 0 ]]; then
|
|
pommods="${pommods} ${module}"
|
|
fi
|
|
done
|
|
|
|
#shellcheck disable=SC2086
|
|
CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
|
|
}
|
|
|
|
## @description git checkout the appropriate branch to test. Additionally, this calls
|
|
## @description 'determine_issue' and 'determine_branch' based upon the context provided
|
|
## @description in ${PATCH_DIR} and in git after checkout.
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @return 0 on success. May exit on failure.
|
|
function git_checkout
|
|
{
|
|
local currentbranch
|
|
local exemptdir
|
|
|
|
big_console_header "Confirming git environment"
|
|
|
|
cd "${BASEDIR}"
|
|
if [[ ! -d .git ]]; then
|
|
hadoop_error "ERROR: ${BASEDIR} is not a git repo."
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
if [[ ${RESETREPO} == "true" ]] ; then
|
|
${GIT} reset --hard
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: git reset is failing"
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
# if PATCH_DIR is in BASEDIR, then we don't want
|
|
# git wiping it out.
|
|
exemptdir=$(relative_patchdir)
|
|
if [[ $? == 1 ]]; then
|
|
${GIT} clean -xdf
|
|
else
|
|
# we do, however, want it emptied of all _files_.
|
|
# we need to leave _directories_ in case we are in
|
|
# re-exec mode (which places a directory full of stuff in it)
|
|
hadoop_debug "Exempting ${exemptdir} from clean"
|
|
rm "${PATCH_DIR}/*" 2>/dev/null
|
|
${GIT} clean -xdf -e "${exemptdir}"
|
|
fi
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: git clean is failing"
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}"
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
determine_branch
|
|
if [[ ${PATCH_BRANCH} =~ ^git ]]; then
|
|
PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
|
|
fi
|
|
|
|
# we need to explicitly fetch in case the
|
|
# git ref hasn't been brought in tree yet
|
|
if [[ ${OFFLINE} == false ]]; then
|
|
${GIT} pull --rebase
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: git pull is failing"
|
|
cleanup_and_exit 1
|
|
fi
|
|
fi
|
|
# forcibly checkout this branch or git ref
|
|
${GIT} checkout --force "${PATCH_BRANCH}"
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
# if we've selected a feature branch that has new changes
|
|
# since our last build, we'll need to rebase to see those changes.
|
|
if [[ ${OFFLINE} == false ]]; then
|
|
${GIT} pull --rebase
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: git pull is failing"
|
|
cleanup_and_exit 1
|
|
fi
|
|
fi
|
|
|
|
else
|
|
|
|
status=$(${GIT} status --porcelain)
|
|
if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then
|
|
hadoop_error "ERROR: --dirty-workspace option not provided."
|
|
hadoop_error "ERROR: can't run in a workspace that contains the following modifications"
|
|
hadoop_error "${status}"
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
determine_branch
|
|
if [[ ${PATCH_BRANCH} =~ ^git ]]; then
|
|
PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
|
|
fi
|
|
|
|
currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD)
|
|
if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then
|
|
echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}."
|
|
echo "WARNING: Continuing anyway..."
|
|
PATCH_BRANCH=${currentbranch}
|
|
fi
|
|
fi
|
|
|
|
determine_issue
|
|
|
|
GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD)
|
|
# shellcheck disable=SC2034
|
|
VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum}
|
|
|
|
if [[ "${ISSUE}" == 'Unknown' ]]; then
|
|
echo "Testing patch on ${PATCH_BRANCH}."
|
|
else
|
|
echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
|
|
fi
|
|
|
|
add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}"
|
|
|
|
if [[ ! -f ${BASEDIR}/pom.xml ]]; then
|
|
hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting."
|
|
add_jira_table -1 pre-patch "Unsupported build system."
|
|
output_to_jira 1
|
|
cleanup_and_exit 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
## @description Confirm the source environment is compilable
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function precheck_without_patch
|
|
{
|
|
local -r mypwd=$(pwd)
|
|
|
|
big_console_header "Pre-patch ${PATCH_BRANCH} Java verification"
|
|
|
|
start_clock
|
|
|
|
verify_needed_test javac
|
|
|
|
if [[ $? == 1 ]]; then
|
|
echo "Compiling ${mypwd}"
|
|
echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
|
|
if [[ $? != 0 ]] ; then
|
|
echo "${PATCH_BRANCH} compilation is broken?"
|
|
add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
|
|
return 1
|
|
fi
|
|
else
|
|
echo "Patch does not appear to need javac tests."
|
|
fi
|
|
|
|
verify_needed_test javadoc
|
|
|
|
if [[ $? == 1 ]]; then
|
|
echo "Javadoc'ing ${mypwd}"
|
|
echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
|
|
if [[ $? != 0 ]] ; then
|
|
echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
|
|
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
|
|
return 1
|
|
fi
|
|
else
|
|
echo "Patch does not appear to need javadoc tests."
|
|
fi
|
|
|
|
verify_needed_test site
|
|
|
|
if [[ $? == 1 ]]; then
|
|
echo "site creation for ${mypwd}"
|
|
echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
|
|
if [[ $? != 0 ]] ; then
|
|
echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
|
|
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
|
|
return 1
|
|
fi
|
|
else
|
|
echo "Patch does not appear to need site tests."
|
|
fi
|
|
|
|
precheck_findbugs
|
|
|
|
if [[ $? != 0 ]] ; then
|
|
return 1
|
|
fi
|
|
|
|
add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
|
|
return 0
|
|
}
|
|
|
|
## @description Confirm the given branch is a member of the list of space
|
|
## @description delimited branches or a git ref
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @param branch
|
|
## @param branchlist
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function verify_valid_branch
|
|
{
|
|
local branches=$1
|
|
local check=$2
|
|
local i
|
|
|
|
# shortcut some common
|
|
# non-resolvable names
|
|
if [[ -z ${check} ]]; then
|
|
return 1
|
|
fi
|
|
|
|
if [[ ${check} == patch ]]; then
|
|
return 1
|
|
fi
|
|
|
|
if [[ ${check} =~ ^git ]]; then
|
|
ref=$(echo "${check}" | cut -f2 -dt)
|
|
count=$(echo "${ref}" | wc -c | tr -d ' ')
|
|
|
|
if [[ ${count} == 8 || ${count} == 41 ]]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
fi
|
|
|
|
for i in ${branches}; do
|
|
if [[ "${i}" == "${check}" ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
## @description Try to guess the branch being tested using a variety of heuristics
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success, with PATCH_BRANCH updated appropriately
|
|
## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT
|
|
function determine_branch
|
|
{
|
|
local allbranches
|
|
local patchnamechunk
|
|
|
|
hadoop_debug "Determine branch"
|
|
|
|
# something has already set this, so move on
|
|
if [[ -n ${PATCH_BRANCH} ]]; then
|
|
return
|
|
fi
|
|
|
|
pushd "${BASEDIR}" > /dev/null
|
|
|
|
# developer mode, existing checkout, whatever
|
|
if [[ "${DIRTY_WORKSPACE}" == true ]];then
|
|
PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD)
|
|
echo "dirty workspace mode; applying against existing branch"
|
|
return
|
|
fi
|
|
|
|
allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g)
|
|
|
|
for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
|
|
hadoop_debug "Determine branch: starting with ${j}"
|
|
# shellcheck disable=SC2016
|
|
patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}')
|
|
|
|
# ISSUE.branch.##.patch
|
|
hadoop_debug "Determine branch: ISSUE.branch.##.patch"
|
|
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. )
|
|
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
|
|
if [[ $? == 0 ]]; then
|
|
return
|
|
fi
|
|
|
|
# ISSUE-branch-##.patch
|
|
hadoop_debug "Determine branch: ISSUE-branch-##.patch"
|
|
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
|
|
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
|
|
if [[ $? == 0 ]]; then
|
|
return
|
|
fi
|
|
|
|
# ISSUE-##.patch.branch
|
|
hadoop_debug "Determine branch: ISSUE-##.patch.branch"
|
|
# shellcheck disable=SC2016
|
|
PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}')
|
|
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
|
|
if [[ $? == 0 ]]; then
|
|
return
|
|
fi
|
|
|
|
# ISSUE-branch.##.patch
|
|
hadoop_debug "Determine branch: ISSUE-branch.##.patch"
|
|
# shellcheck disable=SC2016
|
|
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null)
|
|
verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
|
|
if [[ $? == 0 ]]; then
|
|
return
|
|
fi
|
|
done
|
|
|
|
PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
|
|
|
|
popd >/dev/null
|
|
}
|
|
|
|
## @description Try to guess the issue being tested using a variety of heuristics
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success, with ISSUE updated appropriately
|
|
## @return 1 on failure, with ISSUE updated to "Unknown"
|
|
function determine_issue
|
|
{
|
|
local patchnamechunk
|
|
local maybeissue
|
|
|
|
hadoop_debug "Determine issue"
|
|
|
|
# we can shortcut jenkins
|
|
if [[ ${JENKINS} == true ]]; then
|
|
ISSUE=${PATCH_OR_ISSUE}
|
|
return 0
|
|
fi
|
|
|
|
# shellcheck disable=SC2016
|
|
patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}')
|
|
|
|
maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
|
|
|
|
if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then
|
|
ISSUE=${maybeissue}
|
|
return 0
|
|
fi
|
|
|
|
ISSUE="Unknown"
|
|
return 1
|
|
}
|
|
|
|
## @description Add the given test type
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable yes
|
|
## @param test
|
|
function add_test
|
|
{
|
|
local testname=$1
|
|
|
|
hadoop_debug "Testing against ${testname}"
|
|
|
|
if [[ -z ${NEEDED_TESTS} ]]; then
|
|
hadoop_debug "Setting tests to ${testname}"
|
|
NEEDED_TESTS=${testname}
|
|
elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then
|
|
hadoop_debug "Adding ${testname}"
|
|
NEEDED_TESTS="${NEEDED_TESTS} ${testname}"
|
|
fi
|
|
}
|
|
|
|
## @description Verify if a given test was requested
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable yes
|
|
## @param test
|
|
## @return 1 = yes
|
|
## @return 0 = no
|
|
function verify_needed_test
|
|
{
|
|
local i=$1
|
|
|
|
if [[ ${NEEDED_TESTS} =~ $i ]]; then
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
## @description Use some heuristics to determine which long running
|
|
## @description tests to run
|
|
## @audience private
|
|
## @stability stable
|
|
## @replaceable no
|
|
function determine_needed_tests
|
|
{
|
|
local i
|
|
|
|
for i in ${CHANGED_FILES}; do
|
|
if [[ ${i} =~ src/main/webapp ]]; then
|
|
hadoop_debug "tests/webapp: ${i}"
|
|
elif [[ ${i} =~ \.sh
|
|
|| ${i} =~ \.cmd
|
|
]]; then
|
|
hadoop_debug "tests/shell: ${i}"
|
|
elif [[ ${i} =~ \.md$
|
|
|| ${i} =~ \.md\.vm$
|
|
|| ${i} =~ src/site
|
|
|| ${i} =~ src/main/docs
|
|
]]; then
|
|
hadoop_debug "tests/site: ${i}"
|
|
add_test site
|
|
elif [[ ${i} =~ \.c$
|
|
|| ${i} =~ \.cc$
|
|
|| ${i} =~ \.h$
|
|
|| ${i} =~ \.hh$
|
|
|| ${i} =~ \.proto$
|
|
|| ${i} =~ src/test
|
|
|| ${i} =~ \.cmake$
|
|
|| ${i} =~ CMakeLists.txt
|
|
]]; then
|
|
hadoop_debug "tests/units: ${i}"
|
|
add_test javac
|
|
add_test unit
|
|
elif [[ ${i} =~ pom.xml$
|
|
|| ${i} =~ \.java$
|
|
|| ${i} =~ src/main
|
|
]]; then
|
|
hadoop_debug "tests/javadoc+units: ${i}"
|
|
add_test javadoc
|
|
add_test javac
|
|
add_test unit
|
|
fi
|
|
|
|
if [[ ${i} =~ \.java$ ]]; then
|
|
add_test findbugs
|
|
fi
|
|
|
|
for plugin in ${PLUGINS}; do
|
|
if declare -f ${plugin}_filefilter >/dev/null 2>&1; then
|
|
"${plugin}_filefilter" "${i}"
|
|
fi
|
|
done
|
|
done
|
|
|
|
add_jira_footer "Optional Tests" "${NEEDED_TESTS}"
|
|
}
|
|
|
|
## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the
|
|
## @description necessary work to place it into ${PATCH_DIR}/patch.
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure, may exit
|
|
function locate_patch
|
|
{
|
|
local notSureIfPatch=false
|
|
hadoop_debug "locate patch"
|
|
|
|
if [[ -f ${PATCH_OR_ISSUE} ]]; then
|
|
PATCH_FILE="${PATCH_OR_ISSUE}"
|
|
else
|
|
if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then
|
|
echo "Patch is being downloaded at $(date) from"
|
|
PATCHURL="${PATCH_OR_ISSUE}"
|
|
else
|
|
${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
|
|
|
|
if [[ $? != 0 ]];then
|
|
hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference."
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
|
|
if [[ ${JENKINS} == true ]]; then
|
|
hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
|
|
cleanup_and_exit 1
|
|
else
|
|
hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
|
|
fi
|
|
fi
|
|
|
|
relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*')
|
|
PATCHURL="http://issues.apache.org${relativePatchURL}"
|
|
if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
|
|
notSureIfPatch=true
|
|
fi
|
|
patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*')
|
|
echo "${ISSUE} patch is being downloaded at $(date) from"
|
|
fi
|
|
echo "${PATCHURL}"
|
|
add_jira_footer "Patch URL" "${PATCHURL}"
|
|
${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
|
|
if [[ $? != 0 ]];then
|
|
hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
|
|
cleanup_and_exit 1
|
|
fi
|
|
PATCH_FILE="${PATCH_DIR}/patch"
|
|
fi
|
|
|
|
if [[ ! -f "${PATCH_DIR}/patch" ]]; then
|
|
cp "${PATCH_FILE}" "${PATCH_DIR}/patch"
|
|
if [[ $? == 0 ]] ; then
|
|
echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}"
|
|
else
|
|
hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
|
|
cleanup_and_exit 1
|
|
fi
|
|
fi
|
|
if [[ ${notSureIfPatch} == "true" ]]; then
|
|
guess_patch_file "${PATCH_DIR}/patch"
|
|
if [[ $? != 0 ]]; then
|
|
hadoop_error "ERROR: ${PATCHURL} is not a patch file."
|
|
cleanup_and_exit 1
|
|
else
|
|
hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
|
|
add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
|
|
fi
|
|
fi
|
|
}
|
|
|
|
## @description Given a possible patch file, guess if it's a patch file without using smart-apply-patch
|
|
## @audience private
|
|
## @stability evolving
|
|
## @param path to patch file to test
|
|
## @return 0 we think it's a patch file
|
|
## @return 1 we think it's not a patch file
|
|
function guess_patch_file
|
|
{
|
|
local patch=$1
|
|
local fileOutput
|
|
|
|
hadoop_debug "Trying to guess is ${patch} is a patch file."
|
|
fileOutput=$("${FILE}" "${patch}")
|
|
if [[ $fileOutput =~ \ diff\ ]]; then
|
|
hadoop_debug "file magic says it's a diff."
|
|
return 0
|
|
fi
|
|
fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
|
|
if [[ $? == 0 ]]; then
|
|
hadoop_debug "first line looks like a patch file."
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
## @description Given ${PATCH_DIR}/patch, verify the patch is good using ${BINDIR}/smart-apply-patch.sh
|
|
## @description in dryrun mode.
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function verify_patch_file
|
|
{
|
|
# Before building, check to make sure that the patch is valid
|
|
export PATCH
|
|
|
|
"${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun
|
|
if [[ $? != 0 ]] ; then
|
|
echo "PATCH APPLICATION FAILED"
|
|
add_jira_table -1 patch "The patch command could not apply the patch during dryrun."
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
## @description Given ${PATCH_DIR}/patch, apply the patch using ${BINDIR}/smart-apply-patch.sh
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return exit on failure
|
|
function apply_patch_file
|
|
{
|
|
big_console_header "Applying patch"
|
|
|
|
export PATCH
|
|
"${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch"
|
|
if [[ $? != 0 ]] ; then
|
|
echo "PATCH APPLICATION FAILED"
|
|
((RESULT = RESULT + 1))
|
|
add_jira_table -1 patch "The patch command could not apply the patch."
|
|
output_to_console 1
|
|
output_to_jira 1
|
|
cleanup_and_exit 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
|
|
## @description If this actually patches the files used for the QA process
|
|
## @description under dev-support and its subdirectories, then
|
|
## @description run with the patched version for the test.
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return none; otherwise relaunches
|
|
function check_reexec
|
|
{
|
|
local commentfile=${PATCH_DIR}/tp.${RANDOM}
|
|
|
|
if [[ ${REEXECED} == true ]]; then
|
|
big_console_header "Re-exec mode detected. Continuing."
|
|
return
|
|
fi
|
|
|
|
if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch
|
|
&& ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then
|
|
return
|
|
fi
|
|
|
|
big_console_header "dev-support patch detected"
|
|
|
|
if [[ ${RESETREPO} == false ]]; then
|
|
((RESULT = RESULT + 1))
|
|
hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :("
|
|
add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory."
|
|
return
|
|
fi
|
|
|
|
printf "\n\nRe-executing against patched versions to test.\n\n"
|
|
|
|
apply_patch_file
|
|
|
|
if [[ ${JENKINS} == true ]]; then
|
|
|
|
rm "${commentfile}" 2>/dev/null
|
|
|
|
echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}"
|
|
echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}"
|
|
echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}"
|
|
|
|
write_to_jira "${commentfile}"
|
|
rm "${commentfile}"
|
|
fi
|
|
|
|
cd "${CWD}"
|
|
mkdir -p "${PATCH_DIR}/dev-support-test"
|
|
cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test"
|
|
cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test"
|
|
|
|
big_console_header "exec'ing test-patch.sh now..."
|
|
|
|
exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
|
|
--reexec \
|
|
--branch="${PATCH_BRANCH}" \
|
|
--patch-dir="${PATCH_DIR}" \
|
|
"${USER_PARAMS[@]}"
|
|
}
|
|
|
|
## @description Check the current directory for @author tags
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_author
|
|
{
|
|
local authorTags
|
|
|
|
big_console_header "Checking there are no @author tags in the patch."
|
|
|
|
start_clock
|
|
|
|
if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then
|
|
add_jira_table 0 @author "Skipping @author checks as test-patch has been patched."
|
|
return 0
|
|
fi
|
|
|
|
authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch")
|
|
echo "There appear to be ${authorTags} @author tags in the patch."
|
|
if [[ ${authorTags} != 0 ]] ; then
|
|
add_jira_table -1 @author \
|
|
"The patch appears to contain ${authorTags} @author tags which the Hadoop" \
|
|
" community has agreed to not allow in code contributions."
|
|
return 1
|
|
fi
|
|
add_jira_table +1 @author "The patch does not contain any @author tags."
|
|
return 0
|
|
}
|
|
|
|
## @description Check the patch file for changed/new tests
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_modified_unittests
|
|
{
|
|
local testReferences=0
|
|
local i
|
|
|
|
verify_needed_test unit
|
|
|
|
if [[ $? == 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Checking there are new or changed tests in the patch."
|
|
|
|
start_clock
|
|
|
|
for i in ${CHANGED_FILES}; do
|
|
if [[ ${i} =~ /test/ ]]; then
|
|
((testReferences=testReferences + 1))
|
|
fi
|
|
done
|
|
|
|
echo "There appear to be ${testReferences} test file(s) referenced in the patch."
|
|
if [[ ${testReferences} == 0 ]] ; then
|
|
add_jira_table -1 "tests included" \
|
|
"The patch doesn't appear to include any new or modified tests. " \
|
|
"Please justify why no new tests are needed for this patch." \
|
|
"Also please list what manual steps were performed to verify this patch."
|
|
return 1
|
|
fi
|
|
add_jira_table +1 "tests included" \
|
|
"The patch appears to include ${testReferences} new or modified test files."
|
|
return 0
|
|
}
|
|
|
|
## @description Helper for check_javadoc
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function count_javadoc_warns
|
|
{
|
|
local warningfile=$1
|
|
|
|
#shellcheck disable=SC2016,SC2046
|
|
return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}')
|
|
}
|
|
|
|
## @description Count and compare the number of JavaDoc warnings pre- and post- patch
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_javadoc
|
|
{
|
|
local numBranchJavadocWarnings
|
|
local numPatchJavadocWarnings
|
|
|
|
verify_needed_test javadoc
|
|
|
|
if [[ $? == 0 ]]; then
|
|
echo "This patch does not appear to need javadoc checks."
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Determining number of patched javadoc warnings"
|
|
|
|
start_clock
|
|
|
|
if [[ -d hadoop-project ]]; then
|
|
(cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
|
|
fi
|
|
if [[ -d hadoop-common-project/hadoop-annotations ]]; then
|
|
(cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
|
|
fi
|
|
echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
|
|
count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
|
|
numBranchJavadocWarnings=$?
|
|
count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
|
|
numPatchJavadocWarnings=$?
|
|
|
|
echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch."
|
|
if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then
|
|
if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then
|
|
|
|
${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt"
|
|
${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
|
|
${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \
|
|
"${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \
|
|
> "${PATCH_DIR}/diffJavadocWarnings.txt"
|
|
rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
|
|
|
|
add_jira_table -1 javadoc "The applied patch generated "\
|
|
"$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \
|
|
" additional warning messages."
|
|
add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt"
|
|
return 1
|
|
fi
|
|
fi
|
|
add_jira_table +1 javadoc "There were no new javadoc warning messages."
|
|
return 0
|
|
}
|
|
|
|
## @description Make sure site still compiles
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_site
|
|
{
|
|
local -r mypwd=$(pwd)
|
|
|
|
verify_needed_test site
|
|
|
|
if [[ $? == 0 ]]; then
|
|
echo "This patch does not appear to need site checks."
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Determining if patched site still builds"
|
|
|
|
start_clock
|
|
|
|
echo "site creation for ${mypwd}"
|
|
echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
|
|
if [[ $? != 0 ]] ; then
|
|
echo "Site compilation is broken"
|
|
add_jira_table -1 site "Site compilation is broken."
|
|
add_jira_footer site "@@BASE@@/patchSiteWarnings.txt"
|
|
return 1
|
|
fi
|
|
add_jira_table +1 site "Site still builds."
|
|
return 0
|
|
}
|
|
|
|
## @description Helper for check_javac
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function count_javac_warns
|
|
{
|
|
local warningfile=$1
|
|
#shellcheck disable=SC2016,SC2046
|
|
return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}")
|
|
}
|
|
|
|
## @description Count and compare the number of javac warnings pre- and post- patch
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_javac
|
|
{
|
|
local branchJavacWarnings
|
|
local patchJavacWarnings
|
|
|
|
verify_needed_test javac
|
|
|
|
if [[ $? == 0 ]]; then
|
|
echo "This patch does not appear to need javac checks."
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Determining number of patched javac warnings."
|
|
|
|
start_clock
|
|
|
|
echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
|
|
if [[ $? != 0 ]] ; then
|
|
add_jira_table -1 javac "The patch appears to cause the build to fail."
|
|
return 2
|
|
fi
|
|
### Compare ${PATCH_BRANCH} and patch javac warning numbers
|
|
if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then
|
|
${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
|
|
${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
|
|
|
|
count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
|
|
branchJavacWarnings=$?
|
|
count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
|
|
patchJavacWarnings=$?
|
|
|
|
echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch."
|
|
if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then
|
|
if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then
|
|
|
|
${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \
|
|
"${PATCH_DIR}/filteredPatchJavacWarnings.txt" \
|
|
> "${PATCH_DIR}/diffJavacWarnings.txt"
|
|
|
|
add_jira_table -1 javac "The applied patch generated "\
|
|
"$((patchJavacWarnings-branchJavacWarnings))" \
|
|
" additional warning messages."
|
|
|
|
add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt"
|
|
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
add_jira_table +1 javac "There were no new javac warning messages."
|
|
return 0
|
|
}
|
|
|
|
## @description Verify all files have an Apache License
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_apachelicense
|
|
{
|
|
big_console_header "Determining number of patched release audit warnings."
|
|
|
|
start_clock
|
|
|
|
echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
|
|
#shellcheck disable=SC2038
|
|
find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
|
|
|
|
### Compare ${PATCH_BRANCH} and patch release audit warning numbers
|
|
if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then
|
|
patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt")
|
|
echo ""
|
|
echo ""
|
|
echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch."
|
|
if [[ ${patchReleaseAuditWarnings} != "" ]] ; then
|
|
if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then
|
|
add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings."
|
|
|
|
${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \
|
|
> "${PATCH_DIR}/patchReleaseAuditProblems.txt"
|
|
|
|
echo "Lines that start with ????? in the release audit "\
|
|
"report indicate files that do not have an Apache license header." \
|
|
>> "${PATCH_DIR}/patchReleaseAuditProblems.txt"
|
|
|
|
add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt"
|
|
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings."
|
|
return 0
|
|
}
|
|
|
|
## @description Verify mvn install works
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_mvn_install
|
|
{
|
|
local retval
|
|
|
|
verify_needed_test javadoc
|
|
retval=$?
|
|
|
|
verify_needed_test javac
|
|
((retval = retval + $? ))
|
|
|
|
if [[ ${retval} == 0 ]]; then
|
|
echo "This patch does not appear to need mvn install checks."
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Installing all of the jars"
|
|
|
|
start_clock
|
|
echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
|
|
retval=$?
|
|
if [[ ${retval} != 0 ]]; then
|
|
add_jira_table -1 install "The patch causes mvn install to fail."
|
|
else
|
|
add_jira_table +1 install "mvn install still works."
|
|
fi
|
|
return ${retval}
|
|
}
|
|
|
|
## @description are the needed bits for findbugs present?
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 findbugs will work for our use
|
|
## @return 1 findbugs is missing some component
|
|
function findbugs_is_installed
|
|
{
|
|
if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
|
|
printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
|
|
add_jira_table -1 findbugs "Findbugs is not installed."
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
## @description Run the maven findbugs plugin and record found issues in a bug database
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function findbugs_mvnrunner
|
|
{
|
|
local name=$1
|
|
local logfile=$2
|
|
local warnings_file=$3
|
|
|
|
echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
|
|
"-D${PROJECT_NAME}PatchProcess" < /dev/null
|
|
if [[ $? != 0 ]]; then
|
|
return 1
|
|
fi
|
|
cp target/findbugsXml.xml "${warnings_file}.xml"
|
|
|
|
"${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
|
|
"${warnings_file}.xml" "${warnings_file}.xml"
|
|
if [[ $? != 0 ]]; then
|
|
return 1
|
|
fi
|
|
|
|
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
|
|
"${warnings_file}.html"
|
|
if [[ $? != 0 ]]; then
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
## @description Track pre-existing findbugs warnings
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function precheck_findbugs
|
|
{
|
|
local -r mypwd=$(pwd)
|
|
local module_suffix
|
|
local modules=${CHANGED_MODULES}
|
|
local module
|
|
local findbugs_version
|
|
local rc=0
|
|
local module_findbugs_warnings
|
|
local findbugs_warnings=0
|
|
|
|
verify_needed_test findbugs
|
|
|
|
if [[ $? == 0 ]]; then
|
|
echo "Patch does not appear to need findbugs tests."
|
|
return 0
|
|
fi
|
|
|
|
echo "findbugs baseline for ${mypwd}"
|
|
|
|
findbugs_is_installed
|
|
if [[ $? != 0 ]]; then
|
|
return 1
|
|
fi
|
|
|
|
for module in ${modules}
|
|
do
|
|
pushd "${module}" >/dev/null
|
|
echo " Running findbugs in ${module}"
|
|
module_suffix=$(basename "${module}")
|
|
findbugs_mvnrunner "${PATCH_BRANCH}" \
|
|
"${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
|
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
|
|
(( rc = rc + $? ))
|
|
|
|
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
|
|
#shellcheck disable=SC2016
|
|
module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
|
|
"${PATCH_BRANCH}" \
|
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
|
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
|
|
| ${AWK} '{print $1}')
|
|
if [[ $? != 0 ]]; then
|
|
popd >/dev/null
|
|
return 1
|
|
fi
|
|
|
|
findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
|
|
|
|
if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
|
|
add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
|
|
fi
|
|
fi
|
|
popd >/dev/null
|
|
done
|
|
|
|
#shellcheck disable=SC2016
|
|
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
|
|
|
|
if [[ ${rc} -ne 0 ]]; then
|
|
echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
|
|
add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
|
|
return 1
|
|
fi
|
|
|
|
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
|
|
${findbugs_warnings} -gt 0 ]] ; then
|
|
echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
|
|
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
## @description Verify patch does not trigger any findbugs warnings
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_findbugs
|
|
{
|
|
local rc=0
|
|
local module
|
|
local modules=${CHANGED_MODULES}
|
|
local module_suffix
|
|
local combined_xml
|
|
local newBugs
|
|
local new_findbugs_warnings
|
|
local new_findbugs_fixed_warnings
|
|
local findbugs_warnings=0
|
|
local findbugs_fixed_warnings=0
|
|
local line
|
|
local firstpart
|
|
local secondpart
|
|
local findbugs_version
|
|
|
|
verify_needed_test findbugs
|
|
|
|
if [[ $? == 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Determining number of patched Findbugs warnings."
|
|
|
|
start_clock
|
|
|
|
findbugs_is_installed
|
|
if [[ $? != 0 ]]; then
|
|
return 1
|
|
fi
|
|
|
|
for module in ${modules}
|
|
do
|
|
pushd "${module}" >/dev/null
|
|
echo " Running findbugs in ${module}"
|
|
module_suffix=$(basename "${module}")
|
|
|
|
findbugs_mvnrunner patch \
|
|
"${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
|
|
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
|
|
|
|
if [[ $? != 0 ]] ; then
|
|
((rc = rc +1))
|
|
echo "Post-patch findbugs compilation is broken."
|
|
add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
|
|
continue
|
|
fi
|
|
|
|
combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
|
|
newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
|
|
"${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
|
|
-output "${combined_xml}" \
|
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
|
|
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
|
|
if [[ $? != 0 ]]; then
|
|
popd >/dev/null
|
|
return 1
|
|
fi
|
|
|
|
#shellcheck disable=SC2016
|
|
new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
|
|
"${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
|
|
if [[ $? != 0 ]]; then
|
|
popd >/dev/null
|
|
return 1
|
|
fi
|
|
#shellcheck disable=SC2016
|
|
new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
|
|
"${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
|
|
if [[ $? != 0 ]]; then
|
|
popd >/dev/null
|
|
return 1
|
|
fi
|
|
|
|
echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
|
|
findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
|
|
findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
|
|
|
|
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
|
|
"${newBugs}.html"
|
|
if [[ $? != 0 ]]; then
|
|
popd >/dev/null
|
|
return 1
|
|
fi
|
|
|
|
if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
|
|
populate_test_table FindBugs "module:${module_suffix}"
|
|
while read line; do
|
|
firstpart=$(echo "${line}" | cut -f2 -d:)
|
|
secondpart=$(echo "${line}" | cut -f9- -d' ')
|
|
add_jira_test_table "" "${firstpart}:${secondpart}"
|
|
done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
|
|
|
|
add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
|
|
fi
|
|
|
|
popd >/dev/null
|
|
done
|
|
|
|
#shellcheck disable=SC2016
|
|
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
|
|
|
|
if [[ ${findbugs_warnings} -gt 0 ]] ; then
|
|
add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
|
|
return 1
|
|
fi
|
|
|
|
if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
|
|
add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
|
|
else
|
|
add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
## @description Make sure Maven's eclipse generation works.
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_mvn_eclipse
|
|
{
|
|
big_console_header "Running mvn eclipse:eclipse."
|
|
|
|
verify_needed_test javac
|
|
if [[ $? == 0 ]]; then
|
|
echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse"
|
|
return 0
|
|
fi
|
|
|
|
start_clock
|
|
|
|
echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
|
|
if [[ $? != 0 ]] ; then
|
|
add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
|
|
return 1
|
|
fi
|
|
add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse."
|
|
return 0
|
|
}
|
|
|
|
## @description Utility to push many tests into the failure list
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @param testdesc
|
|
## @param testlist
|
|
function populate_test_table
|
|
{
|
|
local reason=$1
|
|
shift
|
|
local first=""
|
|
local i
|
|
|
|
for i in "$@"; do
|
|
if [[ -z "${first}" ]]; then
|
|
add_jira_test_table "${reason}" "${i}"
|
|
first="${reason}"
|
|
else
|
|
add_jira_test_table " " "${i}"
|
|
fi
|
|
done
|
|
}
|
|
|
|
## @description Run and verify the output of the appropriate unit tests
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function check_unittests
|
|
{
|
|
verify_needed_test unit
|
|
|
|
if [[ $? == 0 ]]; then
|
|
echo "Existing unit tests do not test patched files. Skipping."
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Running unit tests"
|
|
|
|
start_clock
|
|
|
|
local failed_tests=""
|
|
local modules=${CHANGED_MODULES}
|
|
local building_common=0
|
|
local hdfs_modules
|
|
local ordered_modules=""
|
|
local failed_test_builds=""
|
|
local test_timeouts=""
|
|
local test_logfile
|
|
local test_build_result
|
|
local module_test_timeouts=""
|
|
local result
|
|
local totalresult=0
|
|
local module_prefix
|
|
|
|
#
|
|
# If we are building hadoop-hdfs-project, we must build the native component
|
|
# of hadoop-common-project first. In order to accomplish this, we move the
|
|
# hadoop-hdfs subprojects to the end of the list so that common will come
|
|
# first.
|
|
#
|
|
# Of course, we may not be building hadoop-common at all-- in this case, we
|
|
# explicitly insert a mvn compile -Pnative of common, to ensure that the
|
|
# native libraries show up where we need them.
|
|
#
|
|
|
|
for module in ${modules}; do
|
|
if [[ ${module} == hadoop-hdfs-project* ]]; then
|
|
hdfs_modules="${hdfs_modules} ${module}"
|
|
elif [[ ${module} == hadoop-common-project* ]]; then
|
|
ordered_modules="${ordered_modules} ${module}"
|
|
building_common=1
|
|
else
|
|
ordered_modules="${ordered_modules} ${module}"
|
|
fi
|
|
done
|
|
|
|
if [[ -n "${hdfs_modules}" ]]; then
|
|
ordered_modules="${ordered_modules} ${hdfs_modules}"
|
|
if [[ ${building_common} -eq 0 ]]; then
|
|
echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
|
|
echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
|
|
if [[ $? != 0 ]]; then
|
|
add_jira_table -1 "native" "Failed to build the native portion " \
|
|
"of hadoop-common prior to running the unit tests in ${ordered_modules}"
|
|
return 1
|
|
else
|
|
add_jira_table +1 "native" "Pre-build of native portion"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
for module in ${ordered_modules}; do
|
|
result=0
|
|
start_clock
|
|
pushd "${module}" >/dev/null
|
|
module_suffix=$(basename "${module}")
|
|
module_prefix=$(echo "${module}" | cut -f2 -d- )
|
|
|
|
test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
|
|
echo " Running tests in ${module_suffix}"
|
|
# shellcheck disable=2086
|
|
echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess
|
|
test_build_result=$?
|
|
|
|
add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"
|
|
|
|
# shellcheck disable=2016
|
|
module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${test_logfile}")
|
|
if [[ -n "${module_test_timeouts}" ]] ; then
|
|
test_timeouts="${test_timeouts} ${module_test_timeouts}"
|
|
result=1
|
|
fi
|
|
|
|
#shellcheck disable=SC2026,SC2038,SC2016
|
|
module_failed_tests=$(find . -name 'TEST*.xml'\
|
|
| xargs "${GREP}" -l -E "<failure|<error"\
|
|
| ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}')
|
|
|
|
if [[ -n "${module_failed_tests}" ]] ; then
|
|
failed_tests="${failed_tests} ${module_failed_tests}"
|
|
result=1
|
|
fi
|
|
if [[ ${test_build_result} != 0 && -z "${module_failed_tests}" && -z "${module_test_timeouts}" ]] ; then
|
|
failed_test_builds="${failed_test_builds} ${module_suffix}"
|
|
result=1
|
|
fi
|
|
popd >/dev/null
|
|
|
|
if [[ $result == 1 ]]; then
|
|
add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}."
|
|
else
|
|
add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}."
|
|
fi
|
|
|
|
((totalresult = totalresult + result))
|
|
done
|
|
|
|
if [[ -n "${failed_tests}" ]] ; then
|
|
# shellcheck disable=SC2086
|
|
populate_test_table "Failed unit tests" ${failed_tests}
|
|
fi
|
|
if [[ -n "${test_timeouts}" ]] ; then
|
|
# shellcheck disable=SC2086
|
|
populate_test_table "Timed out tests" ${test_timeouts}
|
|
fi
|
|
if [[ -n "${failed_test_builds}" ]] ; then
|
|
# shellcheck disable=SC2086
|
|
populate_test_table "Failed build" ${failed_test_builds}
|
|
fi
|
|
|
|
if [[ ${JENKINS} == true ]]; then
|
|
add_jira_footer "Test Results" "${BUILD_URL}testReport/"
|
|
fi
|
|
|
|
if [[ ${totalresult} -gt 0 ]]; then
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
## @description Print out the finished details on the console
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @param runresult
|
|
## @return 0 on success
|
|
## @return 1 on failure
|
|
function output_to_console
|
|
{
|
|
local result=$1
|
|
shift
|
|
local i
|
|
local ourstring
|
|
local vote
|
|
local subs
|
|
local ela
|
|
local comment
|
|
local commentfile1="${PATCH_DIR}/comment.1"
|
|
local commentfile2="${PATCH_DIR}/comment.2"
|
|
local normaltop
|
|
local line
|
|
local seccoladj=0
|
|
local spcfx=${PATCH_DIR}/spcl.txt
|
|
|
|
if [[ ${result} == 0 ]]; then
|
|
if [[ ${JENKINS} == false ]]; then
|
|
{
|
|
printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
|
|
printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
|
|
printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
|
|
printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
|
|
printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
|
|
printf "ICB8bXwgICB8bXwgIAo"
|
|
} > "${spcfx}"
|
|
fi
|
|
printf "\n\n+1 overall\n\n"
|
|
else
|
|
if [[ ${JENKINS} == false ]]; then
|
|
{
|
|
printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
|
|
printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
|
|
printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
|
|
printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
|
|
printf "ICAgICAgICAgICAgICAgICAK"
|
|
} > "${spcfx}"
|
|
fi
|
|
printf "\n\n-1 overall\n\n"
|
|
fi
|
|
|
|
if [[ -f ${spcfx} ]]; then
|
|
if which base64 >/dev/null 2>&1; then
|
|
base64 --decode "${spcfx}" 2>/dev/null
|
|
elif which openssl >/dev/null 2>&1; then
|
|
openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
|
|
fi
|
|
echo
|
|
echo
|
|
rm "${spcfx}"
|
|
fi
|
|
|
|
seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}")
|
|
if [[ ${seccoladj} -lt 10 ]]; then
|
|
seccoladj=10
|
|
fi
|
|
|
|
seccoladj=$((seccoladj + 2 ))
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
|
|
printf "%s\n" "${JIRA_HEADER[${i}]}"
|
|
((i=i+1))
|
|
done
|
|
|
|
printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
|
|
echo "============================================================================"
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
|
|
ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ')
|
|
vote=$(echo "${ourstring}" | cut -f2 -d\|)
|
|
vote=$(colorstripper "${vote}")
|
|
subs=$(echo "${ourstring}" | cut -f3 -d\|)
|
|
ela=$(echo "${ourstring}" | cut -f4 -d\|)
|
|
comment=$(echo "${ourstring}" | cut -f5 -d\|)
|
|
|
|
echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
|
|
normaltop=$(head -1 "${commentfile1}")
|
|
${SED} -e '1d' "${commentfile1}" > "${commentfile2}"
|
|
|
|
printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
|
|
"${subs}" "${ela}" "${normaltop}"
|
|
while read line; do
|
|
printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}"
|
|
done < "${commentfile2}"
|
|
|
|
((i=i+1))
|
|
rm "${commentfile2}" "${commentfile1}" 2>/dev/null
|
|
done
|
|
|
|
if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
|
|
seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}")
|
|
printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
|
|
ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ')
|
|
vote=$(echo "${ourstring}" | cut -f2 -d\|)
|
|
subs=$(echo "${ourstring}" | cut -f3 -d\|)
|
|
printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
|
|
((i=i+1))
|
|
done
|
|
fi
|
|
|
|
printf "\n\n|| Subsystem || Report/Notes ||\n"
|
|
echo "============================================================================"
|
|
i=0
|
|
|
|
until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
|
|
comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
|
|
${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
|
|
printf "%s\n" "${comment}"
|
|
((i=i+1))
|
|
done
|
|
}
|
|
|
|
## @description Print out the finished details to the JIRA issue
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @param runresult
|
|
function output_to_jira
|
|
{
|
|
local result=$1
|
|
local i
|
|
local commentfile=${PATCH_DIR}/commentfile
|
|
local comment
|
|
|
|
rm "${commentfile}" 2>/dev/null
|
|
|
|
if [[ ${JENKINS} != "true" ]] ; then
|
|
return 0
|
|
fi
|
|
|
|
big_console_header "Adding comment to JIRA"
|
|
|
|
add_jira_footer "Console output" "${BUILD_URL}console"
|
|
|
|
if [[ ${result} == 0 ]]; then
|
|
add_jira_header "(/) *{color:green}+1 overall{color}*"
|
|
else
|
|
add_jira_header "(x) *{color:red}-1 overall{color}*"
|
|
fi
|
|
|
|
|
|
{ echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
|
|
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
|
|
printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}"
|
|
((i=i+1))
|
|
done
|
|
|
|
{ echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
|
|
|
|
echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}"
|
|
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
|
|
printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}"
|
|
((i=i+1))
|
|
done
|
|
|
|
|
|
if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
|
|
{ echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
|
|
|
|
echo "|| Reason || Tests ||" >> "${commentfile}"
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
|
|
printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}"
|
|
((i=i+1))
|
|
done
|
|
fi
|
|
|
|
{ echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
|
|
|
|
echo "|| Subsystem || Report/Notes ||" >> "${commentfile}"
|
|
i=0
|
|
until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
|
|
comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
|
|
${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g")
|
|
printf "%s\n" "${comment}" >> "${commentfile}"
|
|
((i=i+1))
|
|
done
|
|
|
|
printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
|
|
|
|
write_to_jira "${commentfile}"
|
|
}
|
|
|
|
## @description Clean the filesystem as appropriate and then exit
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
## @param runresult
|
|
function cleanup_and_exit
|
|
{
|
|
local result=$1
|
|
|
|
if [[ ${JENKINS} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \
|
|
-e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
|
|
# if PATCH_DIR is already inside BASEDIR, then
|
|
# there is no need to move it since we assume that
|
|
# Jenkins or whatever already knows where it is at
|
|
# since it told us to put it there!
|
|
relative_patchdir >/dev/null
|
|
if [[ $? == 1 ]]; then
|
|
hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
|
|
mv "${PATCH_DIR}" "${BASEDIR}"
|
|
fi
|
|
fi
|
|
big_console_header "Finished build."
|
|
|
|
# shellcheck disable=SC2086
|
|
exit ${result}
|
|
}
|
|
|
|
## @description Driver to execute _postcheckout routines
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
function postcheckout
|
|
{
|
|
local routine
|
|
local plugin
|
|
|
|
for routine in find_java_home verify_patch_file
|
|
do
|
|
verify_patchdir_still_exists
|
|
|
|
hadoop_debug "Running ${routine}"
|
|
${routine}
|
|
|
|
(( RESULT = RESULT + $? ))
|
|
if [[ ${RESULT} != 0 ]] ; then
|
|
output_to_console 1
|
|
output_to_jira 1
|
|
cleanup_and_exit 1
|
|
fi
|
|
done
|
|
|
|
for plugin in ${PLUGINS}; do
|
|
verify_patchdir_still_exists
|
|
|
|
if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then
|
|
|
|
hadoop_debug "Running ${plugin}_postcheckout"
|
|
#shellcheck disable=SC2086
|
|
${plugin}_postcheckout
|
|
|
|
(( RESULT = RESULT + $? ))
|
|
if [[ ${RESULT} != 0 ]] ; then
|
|
output_to_console 1
|
|
output_to_jira 1
|
|
cleanup_and_exit 1
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
## @description Driver to execute _preapply routines
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
function preapply
|
|
{
|
|
local routine
|
|
local plugin
|
|
|
|
for routine in precheck_without_patch check_author \
|
|
check_modified_unittests
|
|
do
|
|
verify_patchdir_still_exists
|
|
|
|
hadoop_debug "Running ${routine}"
|
|
${routine}
|
|
|
|
(( RESULT = RESULT + $? ))
|
|
done
|
|
|
|
for plugin in ${PLUGINS}; do
|
|
verify_patchdir_still_exists
|
|
|
|
if declare -f ${plugin}_preapply >/dev/null 2>&1; then
|
|
|
|
hadoop_debug "Running ${plugin}_preapply"
|
|
#shellcheck disable=SC2086
|
|
${plugin}_preapply
|
|
|
|
(( RESULT = RESULT + $? ))
|
|
fi
|
|
done
|
|
}
|
|
|
|
## @description Driver to execute _postapply routines
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
function postapply
|
|
{
|
|
local routine
|
|
local plugin
|
|
local retval
|
|
|
|
compute_gitdiff "${GITDIFFLINES}"
|
|
|
|
check_javac
|
|
retval=$?
|
|
if [[ ${retval} -gt 1 ]] ; then
|
|
output_to_console 1
|
|
output_to_jira 1
|
|
cleanup_and_exit 1
|
|
fi
|
|
|
|
((RESULT = RESULT + retval))
|
|
|
|
for routine in check_javadoc check_apachelicense check_site
|
|
do
|
|
verify_patchdir_still_exists
|
|
hadoop_debug "Running ${routine}"
|
|
$routine
|
|
|
|
(( RESULT = RESULT + $? ))
|
|
|
|
done
|
|
|
|
for plugin in ${PLUGINS}; do
|
|
verify_patchdir_still_exists
|
|
if declare -f ${plugin}_postapply >/dev/null 2>&1; then
|
|
hadoop_debug "Running ${plugin}_postapply"
|
|
#shellcheck disable=SC2086
|
|
${plugin}_postapply
|
|
(( RESULT = RESULT + $? ))
|
|
fi
|
|
done
|
|
}
|
|
|
|
## @description Driver to execute _postinstall routines
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
function postinstall
|
|
{
|
|
local routine
|
|
local plugin
|
|
|
|
for routine in check_mvn_eclipse check_findbugs
|
|
do
|
|
verify_patchdir_still_exists
|
|
hadoop_debug "Running ${routine}"
|
|
${routine}
|
|
(( RESULT = RESULT + $? ))
|
|
done
|
|
|
|
for plugin in ${PLUGINS}; do
|
|
verify_patchdir_still_exists
|
|
if declare -f ${plugin}_postinstall >/dev/null 2>&1; then
|
|
hadoop_debug "Running ${plugin}_postinstall"
|
|
#shellcheck disable=SC2086
|
|
${plugin}_postinstall
|
|
(( RESULT = RESULT + $? ))
|
|
fi
|
|
done
|
|
|
|
}
|
|
|
|
## @description Driver to execute _tests routines
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
function runtests
|
|
{
|
|
local plugin
|
|
|
|
### Run tests for Jenkins or if explictly asked for by a developer
|
|
if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then
|
|
|
|
verify_patchdir_still_exists
|
|
check_unittests
|
|
|
|
(( RESULT = RESULT + $? ))
|
|
fi
|
|
|
|
for plugin in ${PLUGINS}; do
|
|
verify_patchdir_still_exists
|
|
if declare -f ${plugin}_tests >/dev/null 2>&1; then
|
|
hadoop_debug "Running ${plugin}_tests"
|
|
#shellcheck disable=SC2086
|
|
${plugin}_tests
|
|
(( RESULT = RESULT + $? ))
|
|
fi
|
|
done
|
|
}
|
|
|
|
## @description Import content from test-patch.d and optionally
|
|
## @description from user provided plugin directory
|
|
## @audience private
|
|
## @stability evolving
|
|
## @replaceable no
|
|
function importplugins
|
|
{
|
|
local i
|
|
local files=()
|
|
|
|
if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
|
|
if [[ -d "${BINDIR}/test-patch.d" ]]; then
|
|
files=(${BINDIR}/test-patch.d/*.sh)
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
|
|
hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
|
|
files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
|
|
fi
|
|
|
|
for i in "${files[@]}"; do
|
|
hadoop_debug "Importing ${i}"
|
|
. "${i}"
|
|
done
|
|
}
|
|
|
|
## @description Register test-patch.d plugins
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
function add_plugin
|
|
{
|
|
PLUGINS="${PLUGINS} $1"
|
|
}
|
|
|
|
###############################################################################
|
|
###############################################################################
|
|
###############################################################################
|
|
|
|
big_console_header "Bootstrapping test harness"
|
|
|
|
setup_defaults
|
|
|
|
parse_args "$@"
|
|
|
|
importplugins
|
|
|
|
locate_patch
|
|
|
|
find_changed_files
|
|
|
|
determine_needed_tests
|
|
|
|
# from here on out, we'll be in ${BASEDIR} for cwd
|
|
# routines need to pushd/popd if they change.
|
|
git_checkout
|
|
RESULT=$?
|
|
if [[ ${JENKINS} == "true" ]] ; then
|
|
if [[ ${RESULT} != 0 ]] ; then
|
|
exit 100
|
|
fi
|
|
fi
|
|
|
|
check_reexec
|
|
|
|
postcheckout
|
|
|
|
find_changed_modules
|
|
|
|
preapply
|
|
|
|
apply_patch_file
|
|
|
|
postapply
|
|
|
|
check_mvn_install
|
|
|
|
postinstall
|
|
|
|
runtests
|
|
|
|
close_jira_footer
|
|
|
|
close_jira_table
|
|
|
|
output_to_console ${RESULT}
|
|
output_to_jira ${RESULT}
|
|
cleanup_and_exit ${RESULT}
|