HADOOP-11746. rewrite test-patch.sh (aw)

dev-support/shelldocs.py

@@ -17,6 +17,26 @@ import sys
 import string
 from optparse import OptionParser
 
+asflicense='''
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+'''
+
 def docstrip(key,string):
   string=re.sub("^## @%s " % key ,"",string)
   string=string.lstrip()
@@ -220,8 +240,8 @@ def main():
           funcdef.addreturn(line)
         elif line.startswith('function'):
           funcdef.setname(line)
-          if options.skipprnorep:
+          if options.skipprnorep and \
-            if funcdef.getaudience() == "Private" and \
+            funcdef.getaudience() == "Private" and \
             funcdef.getreplace() == "No":
                pass
           else:
@@ -231,6 +251,7 @@ def main():
   allfuncs=sorted(allfuncs)
 
   outfile=open(options.outfile, "w")
+  outfile.write(asflicense)
   for line in toc(allfuncs):
     outfile.write(line)
 

dev-support/test-patch.d/checkstyle.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin checkstyle
+
+CHECKSTYLE_TIMER=0
+
+# if it ends in an explicit .sh, then this is shell code.
+# if it doesn't have an extension, we assume it is shell code too
+function checkstyle_filefilter
+{
+  local filename=$1
+
+  if [[ ${filename} =~ \.java$ ]]; then
+    add_test checkstyle
+  fi
+}
+
+function checkstyle_preapply
+{
+  verify_needed_test checkstyle
+
+  if [[ $? == 0 ]]; then
+    return 0
+  fi
+
+  big_console_header "checkstyle plugin: prepatch"
+
+  start_clock
+  echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
+  if [[ $? != 0 ]] ; then
+    echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
+    add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken."
+    return 1
+  fi
+
+  cp -p "${BASEDIR}/target/checkstyle-result.xml" \
+    "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml"
+
+  # keep track of how much as elapsed for us already
+  CHECKSTYLE_TIMER=$(stop_clock)
+  return 0
+}
+
+function checkstyle_postapply
+{
+  verify_needed_test checkstyle
+
+  if [[ $? == 0 ]]; then
+    return 0
+  fi
+
+  big_console_header "checkstyle plugin: postpatch"
+
+  start_clock
+
+  # add our previous elapsed to our new timer
+  # by setting the clock back
+  offset_clock "${CHECKSTYLE_TIMER}"
+
+  echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
+  if [[ $? != 0 ]] ; then
+    echo "Post-patch checkstyle compilation is broken."
+    add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken."
+    return 1
+  fi
+
+  cp -p "${BASEDIR}/target/checkstyle-result.xml" \
+    "${PATCH_DIR}/checkstyle-result-patch.xml"
+
+  checkstyle_runcomparison
+
+  # shellcheck disable=SC2016
+  CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}')
+
+  if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then
+
+    add_jira_table -1 checkstyle "The applied patch generated "\
+      "${CHECKSTYLE_POSTPATCH}" \
+      " additional checkstyle issues."
+    add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt"
+
+    return 1
+  fi
+  add_jira_table +1 checkstyle "There were no new checkstyle issues."
+  return 0
+}
+
+
+function checkstyle_runcomparison
+{
+
+  python <(cat <<EOF
+import os
+import sys
+import xml.etree.ElementTree as etree
+from collections import defaultdict
+
+if len(sys.argv) != 3 :
+  print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
+  exit(1)
+
+def path_key(x):
+  path = x.attrib['name']
+  return path[path.find('${PROJECT_NAME}-'):]
+
+def print_row(path, master_errors, patch_errors):
+    print '%s\t%s\t%s' % (k,master_dict[k],child_errors)
+
+master = etree.parse(sys.argv[1])
+patch = etree.parse(sys.argv[2])
+
+master_dict = defaultdict(int)
+
+for child in master.getroot().getchildren():
+    if child.tag != 'file':
+        continue
+    child_errors = len(child.getchildren())
+    if child_errors == 0:
+        continue
+    master_dict[path_key(child)] = child_errors
+
+for child in patch.getroot().getchildren():
+    if child.tag != 'file':
+        continue
+    child_errors = len(child.getchildren())
+    if child_errors == 0:
+        continue
+    k = path_key(child)
+    if child_errors > master_dict[k]:
+        print_row(k, master_dict[k], child_errors)
+
+EOF
+) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt"
+
+}

dev-support/test-patch.d/shellcheck.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin shellcheck
+
+SHELLCHECK_TIMER=0
+
+SHELLCHECK=${SHELLCHECK:-$(which shellcheck)}
+
+SHELLCHECK_SPECIFICFILES=""
+
+# if it ends in an explicit .sh, then this is shell code.
+# if it doesn't have an extension, we assume it is shell code too
+function shellcheck_filefilter
+{
+  local filename=$1
+
+  if [[ ${filename} =~ \.sh$ ]]; then
+    add_test shellcheck
+    SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
+  fi
+
+  if [[ ! ${filename} =~ \. ]]; then
+    add_test shellcheck
+  fi
+}
+
+function shellcheck_private_findbash
+{
+  local i
+  local value
+  local list
+
+  while read line; do
+    value=$(find "${line}" ! -name '*.cmd' -type f \
+      | ${GREP} -E -v '(.orig$|.rej$)')
+    list="${list} ${value}"
+  done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
+  # shellcheck disable=SC2086
+  echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
+}
+
+function shellcheck_preapply
+{
+  local i
+
+  verify_needed_test shellcheck
+  if [[ $? == 0 ]]; then
+    return 0
+  fi
+
+  big_console_header "shellcheck plugin: prepatch"
+
+  if [[ -z "${SHELLCHECK}" ]]; then
+    hadoop_error "shellcheck is not available."
+  fi
+
+  start_clock
+
+  # shellcheck disable=SC2016
+  SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}')
+
+  echo "Running shellcheck against all identifiable shell scripts"
+  pushd "${BASEDIR}" >/dev/null
+  for i in $(shellcheck_private_findbash); do
+    if [[ -f ${i} ]]; then
+      ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
+    fi
+  done
+  popd > /dev/null
+  # keep track of how much as elapsed for us already
+  SHELLCHECK_TIMER=$(stop_clock)
+  return 0
+}
+
+function shellcheck_postapply
+{
+  local i
+
+  verify_needed_test shellcheck
+  if [[ $? == 0 ]]; then
+    return 0
+  fi
+
+  big_console_header "shellcheck plugin: postpatch"
+
+  if [[ -z "${SHELLCHECK}" ]]; then
+    hadoop_error "shellcheck is not available."
+  fi
+
+  start_clock
+
+  # add our previous elapsed to our new timer
+  # by setting the clock back
+  offset_clock "${SHELLCHECK_TIMER}"
+
+  echo "Running shellcheck against all identifiable shell scripts"
+  # we re-check this in case one has been added
+  for i in $(shellcheck_private_findbash); do
+    ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
+  done
+
+  # shellcheck disable=SC2016
+  numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
+  # shellcheck disable=SC2016
+  numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
+
+  ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
+    "${PATCH_DIR}/patchshellcheck-result.txt" \
+      | ${GREP} '^+\.' \
+      > "${PATCH_DIR}/diffpatchshellcheck.txt"
+
+  # shellcheck disable=SC2016
+  diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}')
+
+  if [[ ${diffPostpatch} -gt 0 ]] ; then
+    add_jira_table -1 shellcheck "The applied patch generated "\
+      "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
+    add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
+    return 1
+  fi
+
+  add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
+  return 0
+}

dev-support/test-patch.d/whitespace.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin whitespace
+
+function whitespace_preapply
+{
+  local count
+
+  big_console_header "Checking for whitespace at the end of lines"
+  start_clock
+
+  ${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt"
+
+  # shellcheck disable=SC2016
+  count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
+
+  if [[ ${count} -gt 0 ]]; then
+    add_jira_table -1 whitespace "The patch has ${count}"\
+      " line(s) that end in whitespace."
+    add_jira_footer whitespace "@@BASE@@/whitespace.txt"
+    return 1
+  fi
+
+  add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
+  return 0
+}

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -447,6 +447,8 @@ Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
 
+    HADOOP-11746. rewrite test-patch.sh (aw)
+
   NEW FEATURES
 
     HADOOP-11226. Add a configuration to set ipc.Client's traffic class with