diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh index f91745f4f01..449fc228b0c 100755 --- a/dev-support/smart-apply-patch.sh +++ b/dev-support/smart-apply-patch.sh @@ -11,14 +11,48 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +# +# Determine if the patch file is a git diff file with prefixes. +# These files are generated via "git diff" *without* the --no-prefix option. +# +# We can apply these patches more easily because we know that the a/ and b/ +# prefixes in the "diff" lines stands for the project root directory. +# So we don't have to hunt for the project root. +# And of course, we know that the patch file was generated using git, so we +# know git apply can handle it properly. +# +# Arguments: file name. +# Return: 0 if it is a git diff; 1 otherwise. +# +is_git_diff_with_prefix() { + DIFF_TYPE="unknown" + while read -r line; do + if [[ "$line" =~ ^diff\ ]]; then + if [[ "$line" =~ ^diff\ \-\-git ]]; then + DIFF_TYPE="git" + else + return 1 # All diff lines must be diff --git lines. + fi + fi + if [[ "$line" =~ ^\+\+\+\ ]] || + [[ "$line" =~ ^\-\-\-\ ]]; then + if ! [[ "$line" =~ ^....[ab]/ || "$line" =~ ^..../dev/null ]]; then + return 1 # All +++ and --- lines must start with a/ or b/ or be /dev/null. + fi + fi + done < $1 + [ x$DIFF_TYPE == x"git" ] || return 1 + return 0 # return true (= 0 in bash) +} PATCH_FILE=$1 +DRY_RUN=$2 if [ -z "$PATCH_FILE" ]; then echo usage: $0 patch-file exit 1 fi +TMPDIR=${TMPDIR:-/tmp} PATCH=${PATCH:-patch} # allow overriding patch binary # Cleanup handler for temporary files @@ -31,13 +65,56 @@ trap "cleanup 1" HUP INT QUIT TERM # Allow passing "-" for stdin patches if [ "$PATCH_FILE" == "-" ]; then - PATCH_FILE=/tmp/tmp.in.$$ + PATCH_FILE="$TMPDIR/smart-apply.in.$RANDOM" cat /dev/fd/0 > $PATCH_FILE TOCLEAN="$TOCLEAN $PATCH_FILE" fi +ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' +if [[ ${PATCH_FILE} =~ ^http || ${PATCH_FILE} =~ ${ISSUE_RE} ]]; then + # Allow downloading of patches + PFILE="$TMPDIR/smart-apply.in.$RANDOM" + TOCLEAN="$TOCLEAN $PFILE" + if [[ ${PATCH_FILE} =~ ^http ]]; then + patchURL="${PATCH_FILE}" + else # Get URL of patch from JIRA + wget -q -O "${PFILE}" "http://issues.apache.org/jira/browse/${PATCH_FILE}" + if [[ $? != 0 ]]; then + echo "Unable to determine what ${PATCH_FILE} may reference." 1>&2 + cleanup 1 + elif [[ $(grep -c 'Patch Available' "${PFILE}") == 0 ]]; then + echo "${PATCH_FILE} is not \"Patch Available\". Exiting." 1>&2 + cleanup 1 + fi + relativePatchURL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PFILE}" | grep -v -e 'htm[l]*$' | sort | tail -1 | grep -o '/jira/secure/attachment/[0-9]*/[^"]*') + patchURL="http://issues.apache.org${relativePatchURL}" + fi + if [[ -n $DRY_RUN ]]; then + echo "Downloading ${patchURL}" + fi + wget -q -O "${PFILE}" "${patchURL}" + if [[ $? != 0 ]]; then + echo "${PATCH_FILE} could not be downloaded." 1>&2 + cleanup 1 + fi + PATCH_FILE="${PFILE}" +fi + +# Special case for git-diff patches without --no-prefix +if is_git_diff_with_prefix "$PATCH_FILE"; then + GIT_FLAGS="--binary -p1 -v" + if [[ -z $DRY_RUN ]]; then + GIT_FLAGS="$GIT_FLAGS --stat --apply " + echo Going to apply git patch with: git apply "${GIT_FLAGS}" + else + GIT_FLAGS="$GIT_FLAGS --check " + fi + git apply ${GIT_FLAGS} "${PATCH_FILE}" + exit $? +fi + # Come up with a list of changed files into $TMP -TMP=/tmp/tmp.paths.$$ +TMP="$TMPDIR/smart-apply.paths.$RANDOM" TOCLEAN="$TOCLEAN $TMP" if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then @@ -46,10 +123,10 @@ if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then # is adding new files and they would apply anywhere. So try to guess the # correct place to put those files. - TMP2=/tmp/tmp.paths.2.$$ + TMP2="$TMPDIR/smart-apply.paths.2.$RANDOM" TOCLEAN="$TOCLEAN $TMP2" - egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2 + egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort -u > $TMP2 if [ ! -s $TMP2 ]; then echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting." @@ -77,8 +154,8 @@ if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then sed -i -e 's,^[ab]/,,' $TMP2 fi - PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 | sort | uniq) - + PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 $TMP2 | sort -u) + # if we are at the project root then nothing more to do if [[ -d hadoop-common-project ]]; then echo Looks like this is being run at project root @@ -105,6 +182,11 @@ else cleanup 1; fi +# If this is a dry run then exit instead of applying the patch +if [[ -n $DRY_RUN ]]; then + cleanup 0; +fi + echo Going to apply patch with: $PATCH -p$PLEVEL $PATCH -p$PLEVEL -E < $PATCH_FILE diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3044aafcf76..c2e5fcb8a6f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -78,6 +78,8 @@ Release 2.8.0 - UNRELEASED HADOOP-11917. test-patch.sh should work with ${BASEDIR}/patchprocess setups (aw) + HADOOP-11925. backport trunk's smart-apply-patch.sh to branch-2 (aw) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp