HADOOP-12113. update test-patch branch to latest code (aw)

dev-support/smart-apply-patch.sh

@@ -43,7 +43,9 @@ PATCH=${PATCH:-patch} # allow overriding patch binary
 # Cleanup handler for temporary files
 TOCLEAN=""
 cleanup() {
+  if [[ -n ${TOCLEAN} ]]; then
     rm $TOCLEAN
+  fi
   exit $1
 }
 trap "cleanup 1" HUP INT QUIT TERM

dev-support/test-patch.d/checkstyle.sh

@@ -18,79 +18,126 @@ add_plugin checkstyle
 
 CHECKSTYLE_TIMER=0
 
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
 function checkstyle_filefilter
 {
   local filename=$1
 
+  if [[ ${BUILDTOOL} == maven
+    || ${BUILDTOOL} == ant ]]; then
     if [[ ${filename} =~ \.java$ ]]; then
       add_test checkstyle
     fi
+  fi
 }
 
-function checkstyle_mvnrunner
+function checkstyle_runner
 {
-  local logfile=$1
+  local repostatus=$1
-  local output=$2
   local tmp=${PATCH_DIR}/$$.${RANDOM}
   local j
+  local i=0
+  local fn
+  local savestart=${TIMER}
+  local savestop
+  local output
+  local logfile
+  local repo
+  local modulesuffix
+  local cmd
 
-  "${MVN}" clean test checkstyle:checkstyle -DskipTests \
+  modules_reset
+
+  if [[ ${repostatus} == branch ]]; then
+    repo=${PATCH_BRANCH}
+  else
+    repo="the patch"
+  fi
+
+  #shellcheck disable=SC2153
+  until [[ $i -eq ${#MODULE[@]} ]]; do
+    start_clock
+    fn=$(module_file_fragment "${MODULE[${i}]}")
+    modulesuffix=$(basename "${MODULE[${i}]}")
+    output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt"
+    logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt"
+    pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null
+
+    case ${BUILDTOOL} in
+      maven)
+        cmd="${MVN} ${MAVEN_ARGS[*]} clean test \
+           checkstyle:checkstyle \
           -Dcheckstyle.consoleOutput=true \
-    "-D${PROJECT_NAME}PatchProcess" 2>&1 \
+          ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch"
+      ;;
+      ant)
+        cmd="${ANT}  \
+          -Dcheckstyle.consoleOutput=true \
+          ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \
+          ${ANT_ARGS[*]} checkstyle"
+      ;;
+    esac
+
+    #shellcheck disable=SC2086
+    echo ${cmd} "> ${logfile}"
+    #shellcheck disable=SC2086
+    ${cmd}  2>&1 \
             | tee "${logfile}" \
             | ${GREP} ^/ \
             | ${SED} -e "s,${BASEDIR},.,g" \
                 > "${tmp}"
 
-  # the checkstyle output files are massive, so
+    if [[ $? == 0 ]] ; then
-  # let's reduce the work by filtering out files
+      module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle"
-  # that weren't changed.  Some modules are
+    else
-  # MASSIVE and this can cut the output down to
+      module_status ${i} -1 "${logfile}" "${modulesuffix} in ${repo} failed checkstyle"
-  # by orders of magnitude!!
+      ((result = result + 1))
+    fi
+    savestop=$(stop_clock)
+    #shellcheck disable=SC2034
+    MODULE_STATUS_TIMER[${i}]=${savestop}
+
     for j in ${CHANGED_FILES}; do
       ${GREP} "${j}" "${tmp}" >> "${output}"
     done
 
     rm "${tmp}" 2>/dev/null
+    # shellcheck disable=SC2086
+    popd >/dev/null
+    ((i=i+1))
+  done
+
+  TIMER=${savestart}
+
+  if [[ ${result} -gt 0 ]]; then
+    return 1
+  fi
+  return 0
 }
 
 function checkstyle_preapply
 {
-  local module_suffix
+  local result
-  local modules=${CHANGED_MODULES}
-  local module
 
-  verify_needed_test checkstyle
+  big_console_header "${PATCH_BRANCH} checkstyle"
-
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "checkstyle plugin: prepatch"
 
   start_clock
 
-  for module in ${modules}
+  verify_needed_test checkstyle
-  do
+  if [[ $? == 0 ]]; then
-    pushd "${module}" >/dev/null
+    echo "Patch does not need checkstyle testing."
-    echo "  Running checkstyle in ${module}"
+    return 0
-    module_suffix=$(basename "${module}")
-
-    checkstyle_mvnrunner \
-      "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
-      "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
-
-    if [[ $? != 0 ]] ; then
-      echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
-      add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
   fi
-    popd >/dev/null
+
-  done
+  personality_modules branch checkstyle
+  checkstyle_runner branch
+  result=$?
+  modules_messages branch checkstyle true
 
   # keep track of how much as elapsed for us already
   CHECKSTYLE_TIMER=$(stop_clock)
+  if [[ ${result} != 0 ]]; then
+    return 1
+  fi
   return 0
 }
 
@@ -135,71 +182,71 @@ function checkstyle_calcdiffs
 
 function checkstyle_postapply
 {
-  local rc=0
+  local result
   local module
-  local modules=${CHANGED_MODULES}
+  local fn
-  local module_suffix
+  local i=0
   local numprepatch=0
   local numpostpatch=0
   local diffpostpatch=0
 
-  verify_needed_test checkstyle
+  big_console_header "Patch checkstyle plugin"
 
+  start_clock
+
+  verify_needed_test checkstyle
   if [[ $? == 0 ]]; then
+    echo "Patch does not need checkstyle testing."
     return 0
   fi
 
-  big_console_header "checkstyle plugin: postpatch"
+  personality_modules patch checkstyle
+  checkstyle_runner patch
+  result=$?
 
-  start_clock
 
   # add our previous elapsed to our new timer
   # by setting the clock back
   offset_clock "${CHECKSTYLE_TIMER}"
 
-  for module in ${modules}
+  until [[ $i -eq ${#MODULE[@]} ]]; do
-  do
+    if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
-    pushd "${module}" >/dev/null
+      ((result=result+1))
-    echo "  Running checkstyle in ${module}"
+      ((i=i+1))
-    module_suffix=$(basename "${module}")
-
-    checkstyle_mvnrunner \
-      "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
-      "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
-
-    if [[ $? != 0 ]] ; then
-      ((rc = rc +1))
-      echo "Post-patch checkstyle compilation is broken."
-      add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
       continue
     fi
+    module=${MODULE[$i]}
+    fn=$(module_file_fragment "${module}")
+
+    if [[ ! -f "${PATCH_DIR}/branch-checkstyle-${fn}.txt" ]]; then
+      touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt"
+    fi
 
     #shellcheck disable=SC2016
     diffpostpatch=$(checkstyle_calcdiffs \
-      "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
+      "${PATCH_DIR}/branch-checkstyle-${fn}.txt" \
-      "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
+      "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \
-      "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
+      "${PATCH_DIR}/diff-checkstyle-${fn}.txt" )
 
     if [[ ${diffpostpatch} -gt 0 ]] ; then
-      ((rc = rc + 1))
+      ((result = result + 1))
 
       # shellcheck disable=SC2016
-      numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
+      numprepatch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}')
       # shellcheck disable=SC2016
-      numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
+      numpostpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}')
 
-      add_jira_table -1 checkstyle "The applied patch generated "\
+      module_status ${i} -1 "diff-checkstyle-${fn}.txt" "Patch generated "\
-        "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
+        "${diffpostpatch} new checkstyle issues in "\
-      footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
+        "${module} (total was ${numprepatch}, now ${numpostpatch})."
     fi
-
+    ((i=i+1))
-    popd >/dev/null
   done
 
-  if [[ ${rc} -gt 0 ]] ; then
+  modules_messages patch checkstyle true
-    add_jira_footer checkstyle "${footer}"
+
+  if [[ ${result} != 0 ]]; then
     return 1
   fi
-  add_jira_table +1 checkstyle "There were no new checkstyle issues."
   return 0
 }

dev-support/test-patch.d/shellcheck.sh

@@ -56,6 +56,7 @@ function shellcheck_private_findbash
 function shellcheck_preapply
 {
   local i
+  local msg
 
   verify_needed_test shellcheck
   if [[ $? == 0 ]]; then
@@ -65,7 +66,7 @@ function shellcheck_preapply
   big_console_header "shellcheck plugin: prepatch"
 
   if [[ ! -x "${SHELLCHECK}" ]]; then
-    hadoop_error "shellcheck is not available."
+    yetus_error "shellcheck is not available."
     return 0
   fi
 
@@ -73,6 +74,13 @@ function shellcheck_preapply
 
   # shellcheck disable=SC2016
   SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
+  msg="v${SHELLCHECK_VERSION}"
+
+  if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then
+    msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)"
+  fi
+
+  add_footer_table shellcheck "${msg}"
 
   echo "Running shellcheck against all identifiable shell scripts"
   pushd "${BASEDIR}" >/dev/null
@@ -138,8 +146,8 @@ function shellcheck_postapply
   big_console_header "shellcheck plugin: postpatch"
 
   if [[ ! -x "${SHELLCHECK}" ]]; then
-    hadoop_error "shellcheck is not available."
+    yetus_error "shellcheck is not available."
-    add_jira_table 0 shellcheck "Shellcheck was not available."
+    add_vote_table 0 shellcheck "Shellcheck was not available."
     return 0
   fi
 
@@ -155,8 +163,13 @@ function shellcheck_postapply
     ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
   done
 
+  if [[ ! -f "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" ]]; then
+    touch "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
+  fi
+
   # shellcheck disable=SC2016
   numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
+
   # shellcheck disable=SC2016
   numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
 
@@ -167,12 +180,12 @@ function shellcheck_postapply
     )
 
   if [[ ${diffPostpatch} -gt 0 ]] ; then
-    add_jira_table -1 shellcheck "The applied patch generated "\
+    add_vote_table -1 shellcheck "The applied patch generated "\
       "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
-    add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
+    add_footer_table shellcheck "@@BASE@@/diffpatchshellcheck.txt"
     return 1
   fi
 
-  add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
+  add_vote_table +1 shellcheck "There were no new shellcheck issues."
   return 0
 }

dev-support/test-patch.d/whitespace.sh

@@ -19,28 +19,47 @@ add_plugin whitespace
 function whitespace_postapply
 {
   local count
-  local j
+  local result=0
 
   big_console_header "Checking for whitespace at the end of lines"
   start_clock
 
   pushd "${BASEDIR}" >/dev/null
-  for j in ${CHANGED_FILES}; do
+  # shellcheck disable=SC2016
-    ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
+  ${AWK} '/\t/ {print $0}' \
-  done
+      "${GITDIFFCONTENT}" \
+    | ${GREP} -v Makefile: >> "${PATCH_DIR}/whitespace-tabs.txt"
+
+   ${GREP} -E '[[:blank:]]$' \
+     "${GITDIFFCONTENT}" \
+      >> "${PATCH_DIR}/whitespace-eol.txt"
 
   # shellcheck disable=SC2016
-  count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
+  count=$(wc -l "${PATCH_DIR}/whitespace-eol.txt" | ${AWK} '{print $1}')
 
   if [[ ${count} -gt 0 ]]; then
-    add_jira_table -1 whitespace "The patch has ${count}"\
+    add_vote_table -1 whitespace "The patch has ${count}"\
       " line(s) that end in whitespace. Use git apply --whitespace=fix."
-    add_jira_footer whitespace "@@BASE@@/whitespace.txt"
+    add_footer_table whitespace "@@BASE@@/whitespace-eol.txt"
+    ((result=result+1))
+  fi
+
+  # shellcheck disable=SC2016
+  count=$(wc -l "${PATCH_DIR}/whitespace-tabs.txt" | ${AWK} '{print $1}')
+
+  if [[ ${count} -gt 0 ]]; then
+    add_vote_table -1 whitespace "The patch has ${count}"\
+      " line(s) with tabs."
+    add_footer_table whitespace "@@BASE@@/whitespace-tabs.txt"
+    ((result=result+1))
+  fi
+
+  if [[ ${result} -gt 0 ]]; then
     popd >/dev/null
     return 1
   fi
 
   popd >/dev/null
-  add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
+  add_vote_table +1 whitespace "Patch has no whitespace issues."
   return 0
 }

hadoop-common-project/hadoop-common/CHANGES-HADOOP-12111.txt

@@ -1 +1,3 @@
   Breakdown of HADOOP-12111 sub-tasks:
+
+  HADOOP-12113. update test-patch branch to latest code (aw)