HADOOP-12030. test-patch should only report on newly introduced findbugs warnings. (Sean Busbey via aw)
This commit is contained in:
parent
e6a171e7a2
commit
1a3ccf4724
|
@ -44,6 +44,7 @@ function setup_defaults
|
||||||
LOAD_SYSTEM_PLUGINS=true
|
LOAD_SYSTEM_PLUGINS=true
|
||||||
|
|
||||||
FINDBUGS_HOME=${FINDBUGS_HOME:-}
|
FINDBUGS_HOME=${FINDBUGS_HOME:-}
|
||||||
|
FINDBUGS_WARNINGS_FAIL_PRECHECK=false
|
||||||
ECLIPSE_HOME=${ECLIPSE_HOME:-}
|
ECLIPSE_HOME=${ECLIPSE_HOME:-}
|
||||||
BUILD_NATIVE=${BUILD_NATIVE:-true}
|
BUILD_NATIVE=${BUILD_NATIVE:-true}
|
||||||
PATCH_BRANCH=""
|
PATCH_BRANCH=""
|
||||||
|
@ -585,6 +586,7 @@ function hadoop_usage
|
||||||
echo "--debug If set, then output some extra stuff to stderr"
|
echo "--debug If set, then output some extra stuff to stderr"
|
||||||
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
|
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
|
||||||
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
|
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
|
||||||
|
echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
|
||||||
echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
|
echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
|
||||||
echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
|
echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
|
||||||
echo "--offline Avoid connecting to the Internet"
|
echo "--offline Avoid connecting to the Internet"
|
||||||
|
@ -666,6 +668,9 @@ function parse_args
|
||||||
--findbugs-home=*)
|
--findbugs-home=*)
|
||||||
FINDBUGS_HOME=${i#*=}
|
FINDBUGS_HOME=${i#*=}
|
||||||
;;
|
;;
|
||||||
|
--findbugs-strict-precheck)
|
||||||
|
FINDBUGS_WARNINGS_FAIL_PRECHECK=true
|
||||||
|
;;
|
||||||
--git-cmd=*)
|
--git-cmd=*)
|
||||||
GIT=${i#*=}
|
GIT=${i#*=}
|
||||||
;;
|
;;
|
||||||
|
@ -1060,6 +1065,12 @@ function precheck_without_patch
|
||||||
echo "Patch does not appear to need site tests."
|
echo "Patch does not appear to need site tests."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
precheck_findbugs
|
||||||
|
|
||||||
|
if [[ $? != 0 ]] ; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
|
add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
@ -1838,40 +1849,84 @@ function check_mvn_install
|
||||||
return ${retval}
|
return ${retval}
|
||||||
}
|
}
|
||||||
|
|
||||||
## @description Verify patch does not trigger any findbugs warnings
|
## @description are the needed bits for findbugs present?
|
||||||
|
## @audience private
|
||||||
|
## @stability evolving
|
||||||
|
## @replaceable no
|
||||||
|
## @return 0 findbugs will work for our use
|
||||||
|
## @return 1 findbugs is missing some component
|
||||||
|
function findbugs_is_installed
|
||||||
|
{
|
||||||
|
if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
|
||||||
|
printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
|
||||||
|
add_jira_table -1 findbugs "Findbugs is not installed."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
## @description Run the maven findbugs plugin and record found issues in a bug database
|
||||||
## @audience private
|
## @audience private
|
||||||
## @stability evolving
|
## @stability evolving
|
||||||
## @replaceable no
|
## @replaceable no
|
||||||
## @return 0 on success
|
## @return 0 on success
|
||||||
## @return 1 on failure
|
## @return 1 on failure
|
||||||
function check_findbugs
|
function findbugs_mvnrunner
|
||||||
{
|
{
|
||||||
local findbugs_version
|
local name=$1
|
||||||
local modules=${CHANGED_MODULES}
|
local logfile=$2
|
||||||
local rc=0
|
local warnings_file=$3
|
||||||
|
|
||||||
|
echo_and_redirect "${logfile}" "${MVN}" clean test findbugs:findbugs -DskipTests \
|
||||||
|
"-D${PROJECT_NAME}PatchProcess" < /dev/null
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
cp target/findbugsXml.xml "${warnings_file}.xml"
|
||||||
|
|
||||||
|
"${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
|
||||||
|
"${warnings_file}.xml" "${warnings_file}.xml"
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
|
||||||
|
"${warnings_file}.html"
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
## @description Track pre-existing findbugs warnings
|
||||||
|
## @audience private
|
||||||
|
## @stability evolving
|
||||||
|
## @replaceable no
|
||||||
|
## @return 0 on success
|
||||||
|
## @return 1 on failure
|
||||||
|
function precheck_findbugs
|
||||||
|
{
|
||||||
|
local -r mypwd=$(pwd)
|
||||||
local module_suffix
|
local module_suffix
|
||||||
local findbugsWarnings=0
|
local modules=${CHANGED_MODULES}
|
||||||
local relative_file
|
local module
|
||||||
local newFindbugsWarnings
|
local findbugs_version
|
||||||
local findbugsWarnings
|
local rc=0
|
||||||
local line
|
local module_findbugs_warnings
|
||||||
local firstpart
|
local findbugs_warnings=0
|
||||||
local secondpart
|
|
||||||
|
|
||||||
big_console_header "Determining number of patched Findbugs warnings."
|
|
||||||
|
|
||||||
|
|
||||||
verify_needed_test findbugs
|
verify_needed_test findbugs
|
||||||
|
|
||||||
if [[ $? == 0 ]]; then
|
if [[ $? == 0 ]]; then
|
||||||
echo "Patch does not touch any java files. Skipping findbugs."
|
echo "Patch does not appear to need findbugs tests."
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
start_clock
|
echo "findbugs baseline for ${mypwd}"
|
||||||
|
|
||||||
if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
|
findbugs_is_installed
|
||||||
printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
|
if [[ $? != 0 ]]; then
|
||||||
add_jira_table -1 findbugs "Findbugs is not installed."
|
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -1880,67 +1935,170 @@ function check_findbugs
|
||||||
pushd "${module}" >/dev/null
|
pushd "${module}" >/dev/null
|
||||||
echo " Running findbugs in ${module}"
|
echo " Running findbugs in ${module}"
|
||||||
module_suffix=$(basename "${module}")
|
module_suffix=$(basename "${module}")
|
||||||
echo_and_redirect "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" "${MVN}" clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess \
|
findbugs_mvnrunner "${PATCH_BRANCH}" \
|
||||||
< /dev/null
|
"${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
|
||||||
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
|
||||||
(( rc = rc + $? ))
|
(( rc = rc + $? ))
|
||||||
|
|
||||||
|
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
|
||||||
|
#shellcheck disable=SC2016
|
||||||
|
module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
|
||||||
|
"${PATCH_BRANCH}" \
|
||||||
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
|
||||||
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
|
||||||
|
| ${AWK} '{print $1}')
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
popd >/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
|
||||||
|
|
||||||
|
if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
|
||||||
|
add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
popd >/dev/null
|
||||||
|
done
|
||||||
|
|
||||||
|
#shellcheck disable=SC2016
|
||||||
|
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
|
||||||
|
|
||||||
|
if [[ ${rc} -ne 0 ]]; then
|
||||||
|
echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
|
||||||
|
add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
|
||||||
|
${findbugs_warnings} -gt 0 ]] ; then
|
||||||
|
echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
|
||||||
|
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
## @description Verify patch does not trigger any findbugs warnings
|
||||||
|
## @audience private
|
||||||
|
## @stability evolving
|
||||||
|
## @replaceable no
|
||||||
|
## @return 0 on success
|
||||||
|
## @return 1 on failure
|
||||||
|
function check_findbugs
|
||||||
|
{
|
||||||
|
local rc=0
|
||||||
|
local module
|
||||||
|
local modules=${CHANGED_MODULES}
|
||||||
|
local module_suffix
|
||||||
|
local combined_xml
|
||||||
|
local newBugs
|
||||||
|
local new_findbugs_warnings
|
||||||
|
local new_findbugs_fixed_warnings
|
||||||
|
local findbugs_warnings=0
|
||||||
|
local findbugs_fixed_warnings=0
|
||||||
|
local line
|
||||||
|
local firstpart
|
||||||
|
local secondpart
|
||||||
|
local findbugs_version
|
||||||
|
|
||||||
|
verify_needed_test findbugs
|
||||||
|
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
big_console_header "Determining number of patched Findbugs warnings."
|
||||||
|
|
||||||
|
start_clock
|
||||||
|
|
||||||
|
findbugs_is_installed
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for module in ${modules}
|
||||||
|
do
|
||||||
|
pushd "${module}" >/dev/null
|
||||||
|
echo " Running findbugs in ${module}"
|
||||||
|
module_suffix=$(basename "${module}")
|
||||||
|
|
||||||
|
findbugs_mvnrunner patch \
|
||||||
|
"${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
|
||||||
|
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
|
||||||
|
|
||||||
|
if [[ $? != 0 ]] ; then
|
||||||
|
((rc = rc +1))
|
||||||
|
echo "Post-patch findbugs compilation is broken."
|
||||||
|
add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
|
||||||
|
newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
|
||||||
|
"${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
|
||||||
|
-output "${combined_xml}" \
|
||||||
|
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
|
||||||
|
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
popd >/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
#shellcheck disable=SC2016
|
||||||
|
new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
|
||||||
|
"${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
popd >/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
#shellcheck disable=SC2016
|
||||||
|
new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
|
||||||
|
"${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
popd >/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
|
||||||
|
findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
|
||||||
|
findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
|
||||||
|
|
||||||
|
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
|
||||||
|
"${newBugs}.html"
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
popd >/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
|
||||||
|
populate_test_table FindBugs "module:${module_suffix}"
|
||||||
|
while read line; do
|
||||||
|
firstpart=$(echo "${line}" | cut -f2 -d:)
|
||||||
|
secondpart=$(echo "${line}" | cut -f9- -d' ')
|
||||||
|
add_jira_test_table "" "${firstpart}:${secondpart}"
|
||||||
|
done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
|
||||||
|
|
||||||
|
add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
|
||||||
|
fi
|
||||||
|
|
||||||
popd >/dev/null
|
popd >/dev/null
|
||||||
done
|
done
|
||||||
|
|
||||||
#shellcheck disable=SC2016
|
#shellcheck disable=SC2016
|
||||||
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
|
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
|
||||||
|
|
||||||
if [[ ${rc} -ne 0 ]]; then
|
if [[ ${findbugs_warnings} -gt 0 ]] ; then
|
||||||
add_jira_table -1 findbugs "The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
|
add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
while read file
|
if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
|
||||||
do
|
add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
|
||||||
relative_file=${file#${BASEDIR}/} # strip leading ${BASEDIR} prefix
|
else
|
||||||
if [[ ${relative_file} != "target/findbugsXml.xml" ]]; then
|
add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
|
||||||
module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
|
|
||||||
module_suffix=$(basename "${module_suffix}")
|
|
||||||
fi
|
|
||||||
|
|
||||||
cp "${file}" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
|
|
||||||
|
|
||||||
"${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -timestamp "01/01/2000" \
|
|
||||||
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
|
|
||||||
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
|
|
||||||
|
|
||||||
#shellcheck disable=SC2016
|
|
||||||
newFindbugsWarnings=$("${FINDBUGS_HOME}/bin/filterBugs" \
|
|
||||||
-first "01/01/2000" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
|
|
||||||
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
|
|
||||||
| ${AWK} '{print $1}')
|
|
||||||
|
|
||||||
echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
|
|
||||||
|
|
||||||
findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
|
|
||||||
|
|
||||||
"${FINDBUGS_HOME}/bin/convertXmlToText" -html \
|
|
||||||
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
|
|
||||||
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.html"
|
|
||||||
|
|
||||||
if [[ ${newFindbugsWarnings} -gt 0 ]] ; then
|
|
||||||
populate_test_table FindBugs "module:${module_suffix}"
|
|
||||||
while read line; do
|
|
||||||
firstpart=$(echo "${line}" | cut -f2 -d:)
|
|
||||||
secondpart=$(echo "${line}" | cut -f9- -d' ')
|
|
||||||
add_jira_test_table "" "${firstpart}:${secondpart}"
|
|
||||||
done < <("${FINDBUGS_HOME}/bin/convertXmlToText" \
|
|
||||||
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml")
|
|
||||||
|
|
||||||
add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
|
|
||||||
fi
|
|
||||||
done < <(find "${BASEDIR}" -name findbugsXml.xml)
|
|
||||||
|
|
||||||
if [[ ${findbugsWarnings} -gt 0 ]] ; then
|
|
||||||
add_jira_table -1 findbugs "The patch appears to introduce ${findbugsWarnings} new Findbugs (version ${findbugs_version}) warnings."
|
|
||||||
return 1
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
|
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -132,6 +132,9 @@ Release 2.8.0 - UNRELEASED
|
||||||
HADOOP-11594. Improve the readability of site index of documentation.
|
HADOOP-11594. Improve the readability of site index of documentation.
|
||||||
(Masatake Iwasaki via aajisaka)
|
(Masatake Iwasaki via aajisaka)
|
||||||
|
|
||||||
|
HADOOP-12030. test-patch should only report on newly introduced
|
||||||
|
findbugs warnings. (Sean Busbey via aw)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HADOOP-11785. Reduce the number of listStatus operation in distcp
|
HADOOP-11785. Reduce the number of listStatus operation in distcp
|
||||||
|
|
Loading…
Reference in New Issue