HADOOP-11746. rewrite test-patch.sh (aw)
This commit is contained in:
Allen Wittenauer
parent
997408eaac
commit
73ddb6b4f8
|
|
@ -17,6 +17,26 @@ import sys
|
|||
import string
|
||||
from optparse import OptionParser
|
||||
|
||||
asflicense='''
|
||||
<!---
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
-->
|
||||
'''
|
||||
|
||||
def docstrip(key,string):
|
||||
string=re.sub("^## @%s " % key ,"",string)
|
||||
string=string.lstrip()
|
||||
|
|
@ -220,17 +240,18 @@ def main():
|
|||
funcdef.addreturn(line)
|
||||
elif line.startswith('function'):
|
||||
funcdef.setname(line)
|
||||
if options.skipprnorep:
|
||||
if funcdef.getaudience() == "Private" and \
|
||||
funcdef.getreplace() == "No":
|
||||
if options.skipprnorep and \
|
||||
funcdef.getaudience() == "Private" and \
|
||||
funcdef.getreplace() == "No":
|
||||
pass
|
||||
else:
|
||||
allfuncs.append(funcdef)
|
||||
else:
|
||||
allfuncs.append(funcdef)
|
||||
funcdef=ShellFunction()
|
||||
|
||||
allfuncs=sorted(allfuncs)
|
||||
|
||||
outfile=open(options.outfile, "w")
|
||||
outfile.write(asflicense)
|
||||
for line in toc(allfuncs):
|
||||
outfile.write(line)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
add_plugin checkstyle
|
||||
|
||||
CHECKSTYLE_TIMER=0
|
||||
|
||||
# if it ends in an explicit .sh, then this is shell code.
|
||||
# if it doesn't have an extension, we assume it is shell code too
|
||||
function checkstyle_filefilter
|
||||
{
|
||||
local filename=$1
|
||||
|
||||
if [[ ${filename} =~ \.java$ ]]; then
|
||||
add_test checkstyle
|
||||
fi
|
||||
}
|
||||
|
||||
function checkstyle_preapply
|
||||
{
|
||||
verify_needed_test checkstyle
|
||||
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "checkstyle plugin: prepatch"
|
||||
|
||||
start_clock
|
||||
echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
|
||||
if [[ $? != 0 ]] ; then
|
||||
echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
|
||||
add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken."
|
||||
return 1
|
||||
fi
|
||||
|
||||
cp -p "${BASEDIR}/target/checkstyle-result.xml" \
|
||||
"${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml"
|
||||
|
||||
# keep track of how much as elapsed for us already
|
||||
CHECKSTYLE_TIMER=$(stop_clock)
|
||||
return 0
|
||||
}
|
||||
|
||||
function checkstyle_postapply
|
||||
{
|
||||
verify_needed_test checkstyle
|
||||
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "checkstyle plugin: postpatch"
|
||||
|
||||
start_clock
|
||||
|
||||
# add our previous elapsed to our new timer
|
||||
# by setting the clock back
|
||||
offset_clock "${CHECKSTYLE_TIMER}"
|
||||
|
||||
echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
|
||||
if [[ $? != 0 ]] ; then
|
||||
echo "Post-patch checkstyle compilation is broken."
|
||||
add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken."
|
||||
return 1
|
||||
fi
|
||||
|
||||
cp -p "${BASEDIR}/target/checkstyle-result.xml" \
|
||||
"${PATCH_DIR}/checkstyle-result-patch.xml"
|
||||
|
||||
checkstyle_runcomparison
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}')
|
||||
|
||||
if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then
|
||||
|
||||
add_jira_table -1 checkstyle "The applied patch generated "\
|
||||
"${CHECKSTYLE_POSTPATCH}" \
|
||||
" additional checkstyle issues."
|
||||
add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt"
|
||||
|
||||
return 1
|
||||
fi
|
||||
add_jira_table +1 checkstyle "There were no new checkstyle issues."
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
function checkstyle_runcomparison
|
||||
{
|
||||
|
||||
python <(cat <<EOF
|
||||
import os
|
||||
import sys
|
||||
import xml.etree.ElementTree as etree
|
||||
from collections import defaultdict
|
||||
|
||||
if len(sys.argv) != 3 :
|
||||
print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
|
||||
exit(1)
|
||||
|
||||
def path_key(x):
|
||||
path = x.attrib['name']
|
||||
return path[path.find('${PROJECT_NAME}-'):]
|
||||
|
||||
def print_row(path, master_errors, patch_errors):
|
||||
print '%s\t%s\t%s' % (k,master_dict[k],child_errors)
|
||||
|
||||
master = etree.parse(sys.argv[1])
|
||||
patch = etree.parse(sys.argv[2])
|
||||
|
||||
master_dict = defaultdict(int)
|
||||
|
||||
for child in master.getroot().getchildren():
|
||||
if child.tag != 'file':
|
||||
continue
|
||||
child_errors = len(child.getchildren())
|
||||
if child_errors == 0:
|
||||
continue
|
||||
master_dict[path_key(child)] = child_errors
|
||||
|
||||
for child in patch.getroot().getchildren():
|
||||
if child.tag != 'file':
|
||||
continue
|
||||
child_errors = len(child.getchildren())
|
||||
if child_errors == 0:
|
||||
continue
|
||||
k = path_key(child)
|
||||
if child_errors > master_dict[k]:
|
||||
print_row(k, master_dict[k], child_errors)
|
||||
|
||||
EOF
|
||||
) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt"
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
add_plugin shellcheck
|
||||
|
||||
SHELLCHECK_TIMER=0
|
||||
|
||||
SHELLCHECK=${SHELLCHECK:-$(which shellcheck)}
|
||||
|
||||
SHELLCHECK_SPECIFICFILES=""
|
||||
|
||||
# if it ends in an explicit .sh, then this is shell code.
|
||||
# if it doesn't have an extension, we assume it is shell code too
|
||||
function shellcheck_filefilter
|
||||
{
|
||||
local filename=$1
|
||||
|
||||
if [[ ${filename} =~ \.sh$ ]]; then
|
||||
add_test shellcheck
|
||||
SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
|
||||
fi
|
||||
|
||||
if [[ ! ${filename} =~ \. ]]; then
|
||||
add_test shellcheck
|
||||
fi
|
||||
}
|
||||
|
||||
function shellcheck_private_findbash
|
||||
{
|
||||
local i
|
||||
local value
|
||||
local list
|
||||
|
||||
while read line; do
|
||||
value=$(find "${line}" ! -name '*.cmd' -type f \
|
||||
| ${GREP} -E -v '(.orig$|.rej$)')
|
||||
list="${list} ${value}"
|
||||
done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
|
||||
# shellcheck disable=SC2086
|
||||
echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
|
||||
}
|
||||
|
||||
function shellcheck_preapply
|
||||
{
|
||||
local i
|
||||
|
||||
verify_needed_test shellcheck
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "shellcheck plugin: prepatch"
|
||||
|
||||
if [[ -z "${SHELLCHECK}" ]]; then
|
||||
hadoop_error "shellcheck is not available."
|
||||
fi
|
||||
|
||||
start_clock
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}')
|
||||
|
||||
echo "Running shellcheck against all identifiable shell scripts"
|
||||
pushd "${BASEDIR}" >/dev/null
|
||||
for i in $(shellcheck_private_findbash); do
|
||||
if [[ -f ${i} ]]; then
|
||||
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
|
||||
fi
|
||||
done
|
||||
popd > /dev/null
|
||||
# keep track of how much as elapsed for us already
|
||||
SHELLCHECK_TIMER=$(stop_clock)
|
||||
return 0
|
||||
}
|
||||
|
||||
function shellcheck_postapply
|
||||
{
|
||||
local i
|
||||
|
||||
verify_needed_test shellcheck
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "shellcheck plugin: postpatch"
|
||||
|
||||
if [[ -z "${SHELLCHECK}" ]]; then
|
||||
hadoop_error "shellcheck is not available."
|
||||
fi
|
||||
|
||||
start_clock
|
||||
|
||||
# add our previous elapsed to our new timer
|
||||
# by setting the clock back
|
||||
offset_clock "${SHELLCHECK_TIMER}"
|
||||
|
||||
echo "Running shellcheck against all identifiable shell scripts"
|
||||
# we re-check this in case one has been added
|
||||
for i in $(shellcheck_private_findbash); do
|
||||
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
|
||||
done
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
|
||||
# shellcheck disable=SC2016
|
||||
numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
|
||||
|
||||
${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
|
||||
"${PATCH_DIR}/patchshellcheck-result.txt" \
|
||||
| ${GREP} '^+\.' \
|
||||
> "${PATCH_DIR}/diffpatchshellcheck.txt"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}')
|
||||
|
||||
if [[ ${diffPostpatch} -gt 0 ]] ; then
|
||||
add_jira_table -1 shellcheck "The applied patch generated "\
|
||||
"${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
|
||||
add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
|
||||
return 1
|
||||
fi
|
||||
|
||||
add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
|
||||
return 0
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
add_plugin whitespace
|
||||
|
||||
function whitespace_preapply
|
||||
{
|
||||
local count
|
||||
|
||||
big_console_header "Checking for whitespace at the end of lines"
|
||||
start_clock
|
||||
|
||||
${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
|
||||
|
||||
if [[ ${count} -gt 0 ]]; then
|
||||
add_jira_table -1 whitespace "The patch has ${count}"\
|
||||
" line(s) that end in whitespace."
|
||||
add_jira_footer whitespace "@@BASE@@/whitespace.txt"
|
||||
return 1
|
||||
fi
|
||||
|
||||
add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
|
||||
return 0
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -447,6 +447,8 @@ Release 2.8.0 - UNRELEASED
|
|||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
HADOOP-11746. rewrite test-patch.sh (aw)
|
||||
|
||||
NEW FEATURES
|
||||
|
||||
HADOOP-11226. Add a configuration to set ipc.Client's traffic class with
|
||||
|
|
|
|||
Loading…
Reference in New Issue