HADOOP-11746. rewrite test-patch.sh (aw)
This commit is contained in:
Allen Wittenauer
parent
447f2f699e
commit
b7251723d8
|
|
@ -0,0 +1,149 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
add_plugin checkstyle
|
||||||
|
|
||||||
|
CHECKSTYLE_TIMER=0
|
||||||
|
|
||||||
|
# if it ends in an explicit .sh, then this is shell code.
|
||||||
|
# if it doesn't have an extension, we assume it is shell code too
|
||||||
|
function checkstyle_filefilter
|
||||||
|
{
|
||||||
|
local filename=$1
|
||||||
|
|
||||||
|
if [[ ${filename} =~ \.java$ ]]; then
|
||||||
|
add_test checkstyle
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkstyle_preapply
|
||||||
|
{
|
||||||
|
verify_needed_test checkstyle
|
||||||
|
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
big_console_header "checkstyle plugin: prepatch"
|
||||||
|
|
||||||
|
start_clock
|
||||||
|
echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
|
||||||
|
if [[ $? != 0 ]] ; then
|
||||||
|
echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
|
||||||
|
add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cp -p "${BASEDIR}/target/checkstyle-result.xml" \
|
||||||
|
"${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml"
|
||||||
|
|
||||||
|
# keep track of how much as elapsed for us already
|
||||||
|
CHECKSTYLE_TIMER=$(stop_clock)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkstyle_postapply
|
||||||
|
{
|
||||||
|
verify_needed_test checkstyle
|
||||||
|
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
big_console_header "checkstyle plugin: postpatch"
|
||||||
|
|
||||||
|
start_clock
|
||||||
|
|
||||||
|
# add our previous elapsed to our new timer
|
||||||
|
# by setting the clock back
|
||||||
|
offset_clock "${CHECKSTYLE_TIMER}"
|
||||||
|
|
||||||
|
echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
|
||||||
|
if [[ $? != 0 ]] ; then
|
||||||
|
echo "Post-patch checkstyle compilation is broken."
|
||||||
|
add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cp -p "${BASEDIR}/target/checkstyle-result.xml" \
|
||||||
|
"${PATCH_DIR}/checkstyle-result-patch.xml"
|
||||||
|
|
||||||
|
checkstyle_runcomparison
|
||||||
|
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}')
|
||||||
|
|
||||||
|
if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then
|
||||||
|
|
||||||
|
add_jira_table -1 checkstyle "The applied patch generated "\
|
||||||
|
"${CHECKSTYLE_POSTPATCH}" \
|
||||||
|
" additional checkstyle issues."
|
||||||
|
add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt"
|
||||||
|
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
add_jira_table +1 checkstyle "There were no new checkstyle issues."
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function checkstyle_runcomparison
|
||||||
|
{
|
||||||
|
|
||||||
|
python <(cat <<EOF
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import xml.etree.ElementTree as etree
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
if len(sys.argv) != 3 :
|
||||||
|
print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
def path_key(x):
|
||||||
|
path = x.attrib['name']
|
||||||
|
return path[path.find('${PROJECT_NAME}-'):]
|
||||||
|
|
||||||
|
def print_row(path, master_errors, patch_errors):
|
||||||
|
print '%s\t%s\t%s' % (k,master_dict[k],child_errors)
|
||||||
|
|
||||||
|
master = etree.parse(sys.argv[1])
|
||||||
|
patch = etree.parse(sys.argv[2])
|
||||||
|
|
||||||
|
master_dict = defaultdict(int)
|
||||||
|
|
||||||
|
for child in master.getroot().getchildren():
|
||||||
|
if child.tag != 'file':
|
||||||
|
continue
|
||||||
|
child_errors = len(child.getchildren())
|
||||||
|
if child_errors == 0:
|
||||||
|
continue
|
||||||
|
master_dict[path_key(child)] = child_errors
|
||||||
|
|
||||||
|
for child in patch.getroot().getchildren():
|
||||||
|
if child.tag != 'file':
|
||||||
|
continue
|
||||||
|
child_errors = len(child.getchildren())
|
||||||
|
if child_errors == 0:
|
||||||
|
continue
|
||||||
|
k = path_key(child)
|
||||||
|
if child_errors > master_dict[k]:
|
||||||
|
print_row(k, master_dict[k], child_errors)
|
||||||
|
|
||||||
|
EOF
|
||||||
|
) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt"
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,138 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
add_plugin shellcheck
|
||||||
|
|
||||||
|
SHELLCHECK_TIMER=0
|
||||||
|
|
||||||
|
SHELLCHECK=${SHELLCHECK:-$(which shellcheck)}
|
||||||
|
|
||||||
|
SHELLCHECK_SPECIFICFILES=""
|
||||||
|
|
||||||
|
# if it ends in an explicit .sh, then this is shell code.
|
||||||
|
# if it doesn't have an extension, we assume it is shell code too
|
||||||
|
function shellcheck_filefilter
|
||||||
|
{
|
||||||
|
local filename=$1
|
||||||
|
|
||||||
|
if [[ ${filename} =~ \.sh$ ]]; then
|
||||||
|
add_test shellcheck
|
||||||
|
SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! ${filename} =~ \. ]]; then
|
||||||
|
add_test shellcheck
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function shellcheck_private_findbash
|
||||||
|
{
|
||||||
|
local i
|
||||||
|
local value
|
||||||
|
local list
|
||||||
|
|
||||||
|
while read line; do
|
||||||
|
value=$(find "${line}" ! -name '*.cmd' -type f \
|
||||||
|
| ${GREP} -E -v '(.orig$|.rej$)')
|
||||||
|
list="${list} ${value}"
|
||||||
|
done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
|
||||||
|
}
|
||||||
|
|
||||||
|
function shellcheck_preapply
|
||||||
|
{
|
||||||
|
local i
|
||||||
|
|
||||||
|
verify_needed_test shellcheck
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
big_console_header "shellcheck plugin: prepatch"
|
||||||
|
|
||||||
|
if [[ -z "${SHELLCHECK}" ]]; then
|
||||||
|
hadoop_error "shellcheck is not available."
|
||||||
|
fi
|
||||||
|
|
||||||
|
start_clock
|
||||||
|
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}')
|
||||||
|
|
||||||
|
echo "Running shellcheck against all identifiable shell scripts"
|
||||||
|
pushd "${BASEDIR}" >/dev/null
|
||||||
|
for i in $(shellcheck_private_findbash); do
|
||||||
|
if [[ -f ${i} ]]; then
|
||||||
|
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
popd > /dev/null
|
||||||
|
# keep track of how much as elapsed for us already
|
||||||
|
SHELLCHECK_TIMER=$(stop_clock)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function shellcheck_postapply
|
||||||
|
{
|
||||||
|
local i
|
||||||
|
|
||||||
|
verify_needed_test shellcheck
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
big_console_header "shellcheck plugin: postpatch"
|
||||||
|
|
||||||
|
if [[ -z "${SHELLCHECK}" ]]; then
|
||||||
|
hadoop_error "shellcheck is not available."
|
||||||
|
fi
|
||||||
|
|
||||||
|
start_clock
|
||||||
|
|
||||||
|
# add our previous elapsed to our new timer
|
||||||
|
# by setting the clock back
|
||||||
|
offset_clock "${SHELLCHECK_TIMER}"
|
||||||
|
|
||||||
|
echo "Running shellcheck against all identifiable shell scripts"
|
||||||
|
# we re-check this in case one has been added
|
||||||
|
for i in $(shellcheck_private_findbash); do
|
||||||
|
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
|
||||||
|
done
|
||||||
|
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
|
||||||
|
|
||||||
|
${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
|
||||||
|
"${PATCH_DIR}/patchshellcheck-result.txt" \
|
||||||
|
| ${GREP} '^+\.' \
|
||||||
|
> "${PATCH_DIR}/diffpatchshellcheck.txt"
|
||||||
|
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}')
|
||||||
|
|
||||||
|
if [[ ${diffPostpatch} -gt 0 ]] ; then
|
||||||
|
add_jira_table -1 shellcheck "The applied patch generated "\
|
||||||
|
"${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
|
||||||
|
add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
add_plugin whitespace
|
||||||
|
|
||||||
|
function whitespace_preapply
|
||||||
|
{
|
||||||
|
local count
|
||||||
|
|
||||||
|
big_console_header "Checking for whitespace at the end of lines"
|
||||||
|
start_clock
|
||||||
|
|
||||||
|
${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt"
|
||||||
|
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
|
||||||
|
|
||||||
|
if [[ ${count} -gt 0 ]]; then
|
||||||
|
add_jira_table -1 whitespace "The patch has ${count}"\
|
||||||
|
" line(s) that end in whitespace."
|
||||||
|
add_jira_footer whitespace "@@BASE@@/whitespace.txt"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
|
||||||
|
return 0
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -12,6 +12,8 @@ Release 2.8.0 - UNRELEASED
|
||||||
HADOOP-9477. Add posixGroups support for LDAP groups mapping service.
|
HADOOP-9477. Add posixGroups support for LDAP groups mapping service.
|
||||||
(Dapeng Sun via Yongjun Zhang)
|
(Dapeng Sun via Yongjun Zhang)
|
||||||
|
|
||||||
|
HADOOP-11746. rewrite test-patch.sh (aw)
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
|
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue