From 5858c74a3854015d67eb71c72dc5d74b8c339737 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 17 Aug 2020 20:08:58 -0700 Subject: [PATCH] HADOOP-17205. Move personality file from Yetus to Hadoop repository (#2226) Signed-off-by: Akira Ajisaka (cherry picked from commit fefacf2578e58cf7bf96d920f5b01dba66909ca2) --- Jenkinsfile | 2 +- dev-support/bin/hadoop.sh | 563 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 564 insertions(+), 1 deletion(-) create mode 100755 dev-support/bin/hadoop.sh diff --git a/Jenkinsfile b/Jenkinsfile index 290e771305c..89387dd6d07 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -96,8 +96,8 @@ pipeline { YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") # our project defaults come from a personality file - # which will get loaded automatically by setting the project name YETUS_ARGS+=("--project=hadoop") + YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh") # lots of different output formats YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh new file mode 100755 index 00000000000..88b6005b7be --- /dev/null +++ b/dev-support/bin/hadoop.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# SHELLDOC-IGNORE +# +# Override these to match Apache Hadoop's requirements +personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute" + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO_DEFAULT="apache/hadoop" + + HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} + if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then + HADOOP_HOMEBREW_DIR=/usr/local + fi +} + +function personality_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --hadoop-isal-prefix=*) + delete_parameter "${i}" + ISAL_HOME=${i#*=} + ;; + --hadoop-openssl-prefix=*) + delete_parameter "${i}" + OPENSSL_HOME=${i#*=} + ;; + --hadoop-snappy-prefix=*) + delete_parameter "${i}" + SNAPPY_HOME=${i#*=} + ;; + esac + done +} + +## @description Calculate the actual module ordering +## @audience private +## @stability evolving +## @param ordering +function hadoop_order +{ + declare ordering=$1 + declare hadoopm + + if [[ ${ordering} = normal ]]; then + hadoopm="${CHANGED_MODULES[*]}" + elif [[ ${ordering} = union ]]; then + hadoopm="${CHANGED_UNION_MODULES}" + elif [[ ${ordering} = mvnsrc ]]; then + hadoopm="${MAVEN_SRC_MODULES[*]}" + elif [[ ${ordering} = mvnsrctest ]]; then + hadoopm="${MAVEN_SRCTEST_MODULES[*]}" + else + hadoopm="${ordering}" + fi + echo "${hadoopm}" +} + +## @description Determine if it is safe to run parallel tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_test_parallel +{ + if [[ -f "${BASEDIR}/pom.xml" ]]; then + HADOOP_VERSION=$(grep '' "${BASEDIR}/pom.xml" \ + | head -1 \ + | "${SED}" -e 's|^ *||' -e 's|.*$||' \ + | cut -f1 -d- ) + export HADOOP_VERSION + else + return 1 + fi + + hmajor=${HADOOP_VERSION%%\.*} + hmajorminor=${HADOOP_VERSION%\.*} + hminor=${hmajorminor##*\.} + # ... and just for reference + #hmicro=${HADOOP_VERSION##*\.} + + # Apache Hadoop v2.8.0 was the first one to really + # get working parallel unit tests + if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then + return 1 + fi + + return 0 +} + +## @description Install extra modules for unit tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_unittest_prereqs +{ + declare input=$1 + declare mods + declare need_common=0 + declare building_common=0 + declare module + declare flags + declare fn + + # prior to running unit tests, hdfs needs libhadoop.so built + # if we're building root, then this extra work is moot + + #shellcheck disable=SC2086 + mods=$(hadoop_order ${input}) + + for module in ${mods}; do + if [[ ${module} = hadoop-hdfs-project* ]]; then + need_common=1 + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + building_common=1 + elif [[ ${module} = . ]]; then + return + fi + done + + # Windows builds *ALWAYS* need hadoop-common compiled + case ${OSTYPE} in + Windows_NT|CYGWIN*|MINGW*|MSYS*) + need_common=1 + ;; + esac + + if [[ ${need_common} -eq 1 + && ${building_common} -eq 0 ]]; then + echo "unit test pre-reqs:" + module="hadoop-common-project/hadoop-common" + fn=$(module_file_fragment "${module}") + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + pushd "${BASEDIR}/${module}" >/dev/null || return 1 + # shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} + popd >/dev/null || return 1 + fi +} + +## @description Calculate the flags/settings for yarn-ui v2 build +## @description based upon the OS +## @audience private +## @stability evolving +function yarn_ui2_flag +{ + + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + # Now it only tested on Linux/OSX, don't enable the profile on + # windows until it get verified + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo -Pyarn-ui + ;; + Darwin) + echo -Pyarn-ui + ;; + *) + # Do nothing + ;; + esac +} + +## @description Calculate the flags/settings for native code +## @description based upon the OS +## @audience private +## @stability evolving +function hadoop_native_flags +{ + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + declare -a args + + # Based upon HADOOP-11937 + # + # Some notes: + # + # - getting fuse to compile on anything but Linux + # is always tricky. + # - Darwin assumes homebrew is in use. + # - HADOOP-12027 required for bzip2 on OS X. + # - bzip2 is broken in lots of places + # (the shared library is considered experimental) + # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 + # + + args=("-Drequire.test.libhadoop") + + if [[ -d "${ISAL_HOME}/include" ]]; then + args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") + fi + + if [[ -d "${OPENSSL_HOME}/include" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/") + fi + + if [[ -d "${SNAPPY_HOME}/include" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy") + fi + + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo \ + -Pnative \ + -Drequire.fuse \ + -Drequire.openssl \ + -Drequire.snappy \ + -Drequire.valgrind \ + -Drequire.zstd \ + "${args[@]}" + ;; + Darwin) + echo \ + "${args[@]}" \ + -Pnative \ + -Drequire.snappy \ + -Drequire.openssl + ;; + Windows_NT|CYGWIN*|MINGW*|MSYS*) + echo \ + "${args[@]}" \ + -Drequire.snappy -Drequire.openssl -Pnative-win + ;; + *) + echo \ + "${args[@]}" + ;; + esac +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare extra="" + declare ordering="normal" + declare needflags=false + declare foundbats=false + declare flags + declare fn + declare i + declare hadoopm + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + checkstyle) + ordering="union" + extra="-DskipTests" + ;; + compile) + ordering="union" + extra="-DskipTests" + needflags=true + + # if something in common changed, we build the whole world + if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then + yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " + ordering="." + fi + ;; + distclean) + ordering="." + extra="-DskipTests" + ;; + javadoc) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then + echo "javadoc pre-reqs:" + for i in hadoop-project \ + hadoop-common-project/hadoop-annotations; do + fn=$(module_file_fragment "${i}") + pushd "${BASEDIR}/${i}" >/dev/null || return 1 + echo "cd ${i}" + echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install + popd >/dev/null || return 1 + done + fi + extra="-Pdocs -DskipTests" + ;; + mvneclipse) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + mvninstall) + extra="-DskipTests" + if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then + ordering=. + fi + ;; + mvnsite) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + unit) + if [[ "${BUILDMODE}" = full ]]; then + ordering=mvnsrc + elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ ${TEST_PARALLEL} = "true" ]] ; then + if hadoop_test_parallel; then + extra="-Pparallel-tests" + if [[ -n ${TEST_THREADS:-} ]]; then + extra="${extra} -DtestsThreadCount=${TEST_THREADS}" + fi + fi + fi + needflags=true + hadoop_unittest_prereqs "${ordering}" + + if ! verify_needed_test javac; then + yetus_debug "hadoop: javac not requested" + if ! verify_needed_test native; then + yetus_debug "hadoop: native not requested" + yetus_debug "hadoop: adding -DskipTests to unit test" + extra="-DskipTests" + fi + fi + + for i in "${CHANGED_FILES[@]}"; do + if [[ "${i}" =~ \.bats ]]; then + foundbats=true + fi + done + + if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then + yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" + extra="${extra} -P!shelltest" + else + extra="${extra} -Pshelltest" + fi + ;; + *) + extra="-DskipTests" + ;; + esac + + if [[ ${needflags} = true ]]; then + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + extra="${extra} ${flags}" + fi + + extra="-Ptest-patch ${extra}" + for module in $(hadoop_order ${ordering}); do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +## @description Add tests based upon personality needs +## @audience private +## @stability evolving +## @param filename +function personality_file_tests +{ + declare filename=$1 + + yetus_debug "Using Hadoop-specific personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + add_test shadedclient + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/scripts + || ${filename} =~ src/test/scripts + || ${filename} =~ src/main/bin + || ${filename} =~ shellprofile\.d + || ${filename} =~ src/main/conf + ]]; then + yetus_debug "tests/shell: ${filename}" + add_test mvnsite + add_test unit + elif [[ ${filename} =~ \.md$ + || ${filename} =~ \.md\.vm$ + || ${filename} =~ src/site + ]]; then + yetus_debug "tests/site: ${filename}" + add_test mvnsite + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test cc + add_test mvnsite + add_test javac + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test compile + add_test javac + add_test javadoc + add_test mvninstall + add_test mvnsite + add_test unit + add_test shadedclient + fi + + # if we change anything in here, e.g. the test scripts + # then run the client artifact tests + if [[ ${filename} =~ hadoop-client-modules ]]; then + add_test shadedclient + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests: src/test" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +## @description Image to print on success +## @audience private +## @stability evolving +function hadoop_console_success +{ + printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; + printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; + printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; + printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; + printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; + printf "ICB8bXwgICB8bXwgIAo" +} + +################################################### +# Hadoop project specific check of IT for shaded artifacts + +add_test_type shadedclient + +## @description check for test modules and add test/plugins as needed +## @audience private +## @stability evolving +function shadedclient_initialize +{ + maven_add_install shadedclient +} + +## @description build client facing shaded artifacts and test them +## @audience private +## @stability evolving +## @param repostatus +function shadedclient_rebuild +{ + declare repostatus=$1 + declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt" + declare module + declare -a modules=() + + if [[ ${OSTYPE} = Windows_NT || + ${OSTYPE} =~ ^CYGWIN.* || + ${OSTYPE} =~ ^MINGW32.* || + ${OSTYPE} =~ ^MSYS.* ]]; then + echo "hadoop personality: building on windows, skipping check of client artifacts." + return 0 + fi + + yetus_debug "hadoop personality: seeing if we need the test of client artifacts." + for module in hadoop-client-modules/hadoop-client-check-invariants \ + hadoop-client-modules/hadoop-client-check-test-invariants \ + hadoop-client-modules/hadoop-client-integration-tests; do + if [ -d "${module}" ]; then + yetus_debug "hadoop personality: test module '${module}' is present." + modules+=(-pl "${module}") + fi + done + if [ ${#modules[@]} -eq 0 ]; then + echo "hadoop personality: no test modules present, skipping check of client artifacts." + return 0 + fi + + big_console_header "Checking client artifacts on ${repostatus}" + + echo_and_redirect "${logfile}" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ + "${modules[@]}" \ + -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true + + count=$("${GREP}" -c '\[ERROR\]' "${logfile}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts." + return 1 + fi + + add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts." + return 0 +}