HADOOP-12113. update test-patch branch to latest code (aw)
This commit is contained in:
Allen Wittenauer
parent
214ac3e931
commit
09a2e360f2
|
|
@ -43,7 +43,9 @@ PATCH=${PATCH:-patch} # allow overriding patch binary
|
|||
# Cleanup handler for temporary files
|
||||
TOCLEAN=""
|
||||
cleanup() {
|
||||
rm $TOCLEAN
|
||||
if [[ -n ${TOCLEAN} ]]; then
|
||||
rm $TOCLEAN
|
||||
fi
|
||||
exit $1
|
||||
}
|
||||
trap "cleanup 1" HUP INT QUIT TERM
|
||||
|
|
|
|||
|
|
@ -18,79 +18,126 @@ add_plugin checkstyle
|
|||
|
||||
CHECKSTYLE_TIMER=0
|
||||
|
||||
# if it ends in an explicit .sh, then this is shell code.
|
||||
# if it doesn't have an extension, we assume it is shell code too
|
||||
function checkstyle_filefilter
|
||||
{
|
||||
local filename=$1
|
||||
|
||||
if [[ ${filename} =~ \.java$ ]]; then
|
||||
add_test checkstyle
|
||||
if [[ ${BUILDTOOL} == maven
|
||||
|| ${BUILDTOOL} == ant ]]; then
|
||||
if [[ ${filename} =~ \.java$ ]]; then
|
||||
add_test checkstyle
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function checkstyle_mvnrunner
|
||||
function checkstyle_runner
|
||||
{
|
||||
local logfile=$1
|
||||
local output=$2
|
||||
local repostatus=$1
|
||||
local tmp=${PATCH_DIR}/$$.${RANDOM}
|
||||
local j
|
||||
local i=0
|
||||
local fn
|
||||
local savestart=${TIMER}
|
||||
local savestop
|
||||
local output
|
||||
local logfile
|
||||
local repo
|
||||
local modulesuffix
|
||||
local cmd
|
||||
|
||||
"${MVN}" clean test checkstyle:checkstyle -DskipTests \
|
||||
-Dcheckstyle.consoleOutput=true \
|
||||
"-D${PROJECT_NAME}PatchProcess" 2>&1 \
|
||||
| tee "${logfile}" \
|
||||
| ${GREP} ^/ \
|
||||
| ${SED} -e "s,${BASEDIR},.,g" \
|
||||
> "${tmp}"
|
||||
modules_reset
|
||||
|
||||
# the checkstyle output files are massive, so
|
||||
# let's reduce the work by filtering out files
|
||||
# that weren't changed. Some modules are
|
||||
# MASSIVE and this can cut the output down to
|
||||
# by orders of magnitude!!
|
||||
for j in ${CHANGED_FILES}; do
|
||||
${GREP} "${j}" "${tmp}" >> "${output}"
|
||||
if [[ ${repostatus} == branch ]]; then
|
||||
repo=${PATCH_BRANCH}
|
||||
else
|
||||
repo="the patch"
|
||||
fi
|
||||
|
||||
#shellcheck disable=SC2153
|
||||
until [[ $i -eq ${#MODULE[@]} ]]; do
|
||||
start_clock
|
||||
fn=$(module_file_fragment "${MODULE[${i}]}")
|
||||
modulesuffix=$(basename "${MODULE[${i}]}")
|
||||
output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt"
|
||||
logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt"
|
||||
pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null
|
||||
|
||||
case ${BUILDTOOL} in
|
||||
maven)
|
||||
cmd="${MVN} ${MAVEN_ARGS[*]} clean test \
|
||||
checkstyle:checkstyle \
|
||||
-Dcheckstyle.consoleOutput=true \
|
||||
${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch"
|
||||
;;
|
||||
ant)
|
||||
cmd="${ANT} \
|
||||
-Dcheckstyle.consoleOutput=true \
|
||||
${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \
|
||||
${ANT_ARGS[*]} checkstyle"
|
||||
;;
|
||||
esac
|
||||
|
||||
#shellcheck disable=SC2086
|
||||
echo ${cmd} "> ${logfile}"
|
||||
#shellcheck disable=SC2086
|
||||
${cmd} 2>&1 \
|
||||
| tee "${logfile}" \
|
||||
| ${GREP} ^/ \
|
||||
| ${SED} -e "s,${BASEDIR},.,g" \
|
||||
> "${tmp}"
|
||||
|
||||
if [[ $? == 0 ]] ; then
|
||||
module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle"
|
||||
else
|
||||
module_status ${i} -1 "${logfile}" "${modulesuffix} in ${repo} failed checkstyle"
|
||||
((result = result + 1))
|
||||
fi
|
||||
savestop=$(stop_clock)
|
||||
#shellcheck disable=SC2034
|
||||
MODULE_STATUS_TIMER[${i}]=${savestop}
|
||||
|
||||
for j in ${CHANGED_FILES}; do
|
||||
${GREP} "${j}" "${tmp}" >> "${output}"
|
||||
done
|
||||
|
||||
rm "${tmp}" 2>/dev/null
|
||||
# shellcheck disable=SC2086
|
||||
popd >/dev/null
|
||||
((i=i+1))
|
||||
done
|
||||
|
||||
rm "${tmp}" 2>/dev/null
|
||||
TIMER=${savestart}
|
||||
|
||||
if [[ ${result} -gt 0 ]]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
function checkstyle_preapply
|
||||
{
|
||||
local module_suffix
|
||||
local modules=${CHANGED_MODULES}
|
||||
local module
|
||||
local result
|
||||
|
||||
verify_needed_test checkstyle
|
||||
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "checkstyle plugin: prepatch"
|
||||
big_console_header "${PATCH_BRANCH} checkstyle"
|
||||
|
||||
start_clock
|
||||
|
||||
for module in ${modules}
|
||||
do
|
||||
pushd "${module}" >/dev/null
|
||||
echo " Running checkstyle in ${module}"
|
||||
module_suffix=$(basename "${module}")
|
||||
verify_needed_test checkstyle
|
||||
if [[ $? == 0 ]]; then
|
||||
echo "Patch does not need checkstyle testing."
|
||||
return 0
|
||||
fi
|
||||
|
||||
checkstyle_mvnrunner \
|
||||
"${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
|
||||
|
||||
if [[ $? != 0 ]] ; then
|
||||
echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
|
||||
add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
|
||||
fi
|
||||
popd >/dev/null
|
||||
done
|
||||
personality_modules branch checkstyle
|
||||
checkstyle_runner branch
|
||||
result=$?
|
||||
modules_messages branch checkstyle true
|
||||
|
||||
# keep track of how much as elapsed for us already
|
||||
CHECKSTYLE_TIMER=$(stop_clock)
|
||||
if [[ ${result} != 0 ]]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
|
|
@ -135,71 +182,71 @@ function checkstyle_calcdiffs
|
|||
|
||||
function checkstyle_postapply
|
||||
{
|
||||
local rc=0
|
||||
local result
|
||||
local module
|
||||
local modules=${CHANGED_MODULES}
|
||||
local module_suffix
|
||||
local fn
|
||||
local i=0
|
||||
local numprepatch=0
|
||||
local numpostpatch=0
|
||||
local diffpostpatch=0
|
||||
|
||||
verify_needed_test checkstyle
|
||||
big_console_header "Patch checkstyle plugin"
|
||||
|
||||
start_clock
|
||||
|
||||
verify_needed_test checkstyle
|
||||
if [[ $? == 0 ]]; then
|
||||
echo "Patch does not need checkstyle testing."
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "checkstyle plugin: postpatch"
|
||||
personality_modules patch checkstyle
|
||||
checkstyle_runner patch
|
||||
result=$?
|
||||
|
||||
start_clock
|
||||
|
||||
# add our previous elapsed to our new timer
|
||||
# by setting the clock back
|
||||
offset_clock "${CHECKSTYLE_TIMER}"
|
||||
|
||||
for module in ${modules}
|
||||
do
|
||||
pushd "${module}" >/dev/null
|
||||
echo " Running checkstyle in ${module}"
|
||||
module_suffix=$(basename "${module}")
|
||||
|
||||
checkstyle_mvnrunner \
|
||||
"${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
|
||||
|
||||
if [[ $? != 0 ]] ; then
|
||||
((rc = rc +1))
|
||||
echo "Post-patch checkstyle compilation is broken."
|
||||
add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
|
||||
until [[ $i -eq ${#MODULE[@]} ]]; do
|
||||
if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
|
||||
((result=result+1))
|
||||
((i=i+1))
|
||||
continue
|
||||
fi
|
||||
module=${MODULE[$i]}
|
||||
fn=$(module_file_fragment "${module}")
|
||||
|
||||
if [[ ! -f "${PATCH_DIR}/branch-checkstyle-${fn}.txt" ]]; then
|
||||
touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt"
|
||||
fi
|
||||
|
||||
#shellcheck disable=SC2016
|
||||
diffpostpatch=$(checkstyle_calcdiffs \
|
||||
"${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
|
||||
"${PATCH_DIR}/branch-checkstyle-${fn}.txt" \
|
||||
"${PATCH_DIR}/patch-checkstyle-${fn}.txt" \
|
||||
"${PATCH_DIR}/diff-checkstyle-${fn}.txt" )
|
||||
|
||||
if [[ ${diffpostpatch} -gt 0 ]] ; then
|
||||
((rc = rc + 1))
|
||||
((result = result + 1))
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
|
||||
numprepatch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}')
|
||||
# shellcheck disable=SC2016
|
||||
numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
|
||||
numpostpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}')
|
||||
|
||||
add_jira_table -1 checkstyle "The applied patch generated "\
|
||||
"${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
|
||||
footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
|
||||
module_status ${i} -1 "diff-checkstyle-${fn}.txt" "Patch generated "\
|
||||
"${diffpostpatch} new checkstyle issues in "\
|
||||
"${module} (total was ${numprepatch}, now ${numpostpatch})."
|
||||
fi
|
||||
|
||||
popd >/dev/null
|
||||
((i=i+1))
|
||||
done
|
||||
|
||||
if [[ ${rc} -gt 0 ]] ; then
|
||||
add_jira_footer checkstyle "${footer}"
|
||||
modules_messages patch checkstyle true
|
||||
|
||||
if [[ ${result} != 0 ]]; then
|
||||
return 1
|
||||
fi
|
||||
add_jira_table +1 checkstyle "There were no new checkstyle issues."
|
||||
return 0
|
||||
}
|
||||
|
|
@ -56,6 +56,7 @@ function shellcheck_private_findbash
|
|||
function shellcheck_preapply
|
||||
{
|
||||
local i
|
||||
local msg
|
||||
|
||||
verify_needed_test shellcheck
|
||||
if [[ $? == 0 ]]; then
|
||||
|
|
@ -65,7 +66,7 @@ function shellcheck_preapply
|
|||
big_console_header "shellcheck plugin: prepatch"
|
||||
|
||||
if [[ ! -x "${SHELLCHECK}" ]]; then
|
||||
hadoop_error "shellcheck is not available."
|
||||
yetus_error "shellcheck is not available."
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
|
@ -73,6 +74,13 @@ function shellcheck_preapply
|
|||
|
||||
# shellcheck disable=SC2016
|
||||
SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
|
||||
msg="v${SHELLCHECK_VERSION}"
|
||||
|
||||
if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then
|
||||
msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)"
|
||||
fi
|
||||
|
||||
add_footer_table shellcheck "${msg}"
|
||||
|
||||
echo "Running shellcheck against all identifiable shell scripts"
|
||||
pushd "${BASEDIR}" >/dev/null
|
||||
|
|
@ -138,8 +146,8 @@ function shellcheck_postapply
|
|||
big_console_header "shellcheck plugin: postpatch"
|
||||
|
||||
if [[ ! -x "${SHELLCHECK}" ]]; then
|
||||
hadoop_error "shellcheck is not available."
|
||||
add_jira_table 0 shellcheck "Shellcheck was not available."
|
||||
yetus_error "shellcheck is not available."
|
||||
add_vote_table 0 shellcheck "Shellcheck was not available."
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
|
@ -155,8 +163,13 @@ function shellcheck_postapply
|
|||
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
|
||||
done
|
||||
|
||||
if [[ ! -f "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" ]]; then
|
||||
touch "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
|
||||
|
||||
|
|
@ -167,12 +180,12 @@ function shellcheck_postapply
|
|||
)
|
||||
|
||||
if [[ ${diffPostpatch} -gt 0 ]] ; then
|
||||
add_jira_table -1 shellcheck "The applied patch generated "\
|
||||
add_vote_table -1 shellcheck "The applied patch generated "\
|
||||
"${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
|
||||
add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
|
||||
add_footer_table shellcheck "@@BASE@@/diffpatchshellcheck.txt"
|
||||
return 1
|
||||
fi
|
||||
|
||||
add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
|
||||
add_vote_table +1 shellcheck "There were no new shellcheck issues."
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,28 +19,47 @@ add_plugin whitespace
|
|||
function whitespace_postapply
|
||||
{
|
||||
local count
|
||||
local j
|
||||
local result=0
|
||||
|
||||
big_console_header "Checking for whitespace at the end of lines"
|
||||
start_clock
|
||||
|
||||
pushd "${BASEDIR}" >/dev/null
|
||||
for j in ${CHANGED_FILES}; do
|
||||
${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
|
||||
done
|
||||
# shellcheck disable=SC2016
|
||||
${AWK} '/\t/ {print $0}' \
|
||||
"${GITDIFFCONTENT}" \
|
||||
| ${GREP} -v Makefile: >> "${PATCH_DIR}/whitespace-tabs.txt"
|
||||
|
||||
${GREP} -E '[[:blank:]]$' \
|
||||
"${GITDIFFCONTENT}" \
|
||||
>> "${PATCH_DIR}/whitespace-eol.txt"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
|
||||
count=$(wc -l "${PATCH_DIR}/whitespace-eol.txt" | ${AWK} '{print $1}')
|
||||
|
||||
if [[ ${count} -gt 0 ]]; then
|
||||
add_jira_table -1 whitespace "The patch has ${count}"\
|
||||
add_vote_table -1 whitespace "The patch has ${count}"\
|
||||
" line(s) that end in whitespace. Use git apply --whitespace=fix."
|
||||
add_jira_footer whitespace "@@BASE@@/whitespace.txt"
|
||||
add_footer_table whitespace "@@BASE@@/whitespace-eol.txt"
|
||||
((result=result+1))
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
count=$(wc -l "${PATCH_DIR}/whitespace-tabs.txt" | ${AWK} '{print $1}')
|
||||
|
||||
if [[ ${count} -gt 0 ]]; then
|
||||
add_vote_table -1 whitespace "The patch has ${count}"\
|
||||
" line(s) with tabs."
|
||||
add_footer_table whitespace "@@BASE@@/whitespace-tabs.txt"
|
||||
((result=result+1))
|
||||
fi
|
||||
|
||||
if [[ ${result} -gt 0 ]]; then
|
||||
popd >/dev/null
|
||||
return 1
|
||||
fi
|
||||
|
||||
popd >/dev/null
|
||||
add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
|
||||
add_vote_table +1 whitespace "Patch has no whitespace issues."
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1 +1,3 @@
|
|||
Breakdown of HADOOP-12111 sub-tasks:
|
||||
|
||||
HADOOP-12113. update test-patch branch to latest code (aw)
|
||||
|
|
|
|||
Loading…
Reference in New Issue