diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 09f2778c7dd..f287f01f637 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -34,7 +34,6 @@ pipeline { YETUS_RELEASE = '0.12.0' // where we'll write everything from different steps. Need a copy here so the final step can check for success/failure. OUTPUT_DIR_RELATIVE_GENERAL = 'output-general' - OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7' OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2' OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3' OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3' @@ -186,7 +185,6 @@ pipeline { // stash with given name for all tests we might run, so that we can unstash all of them even if // we skip some due to e.g. branch-specific JDK or Hadoop support stash name: 'general-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_GENERAL}/doesn't-match" - stash name: 'jdk7-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK7}/doesn't-match" stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match" stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match" stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match" @@ -296,121 +294,14 @@ pipeline { } } } - stage ('yetus jdk7 checks') { + stage ('yetus jdk8 hadoop2 checks') { agent { node { label 'hbase' } } when { - branch 'branch-1*' - } - environment { - BASEDIR = "${env.WORKSPACE}/component" - TESTS = "${env.DEEP_CHECKS}" - OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}" - OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}" - SET_JAVA_HOME = "/usr/lib/jvm/java-7" - } - steps { - // Must do prior to anything else, since if one of them timesout we'll stash the commentfile - sh '''#!/usr/bin/env bash - set -e - rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" - echo '(x) {color:red}-1 jdk7 checks{color}' >"${OUTPUT_DIR}/commentfile" - echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" - ''' - unstash 'yetus' - dir('component') { - checkout scm - } - sh '''#!/usr/bin/env bash - set -e - rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" - "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" - echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" - ls -lh "${OUTPUT_DIR_RELATIVE}/machine" - ''' - script { - def ret = sh( - returnStatus: true, - script: '''#!/usr/bin/env bash - set -e - declare -i status=0 - if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then - echo '(/) {color:green}+1 jdk7 checks{color}' > "${OUTPUT_DIR}/commentfile" - else - echo '(x) {color:red}-1 jdk7 checks{color}' > "${OUTPUT_DIR}/commentfile" - status=1 - fi - echo "-- For more information [see jdk7 report|${BUILD_URL}/JDK7_20Nightly_20Build_20Report/]" >> "${OUTPUT_DIR}/commentfile" - exit "${status}" - ''' - ) - if (ret != 0) { - // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of - // test output. See HBASE-26339 for more details. - currentBuild.result = 'UNSTABLE' - } - } - } - post { - always { - stash name: 'jdk7-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" - junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true - // zip surefire reports. - sh '''#!/bin/bash -e - if [ -d "${OUTPUT_DIR}/archiver" ]; then - count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) - if [[ 0 -ne ${count} ]]; then - echo "zipping ${count} archived files" - zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" - else - echo "No archived files, skipping compressing." - fi - else - echo "No archiver directory, skipping compressing." - fi -''' - sshPublisher(publishers: [ - sshPublisherDesc(configName: 'Nightlies', - transfers: [ - sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", - sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip" - ) - ] - ) - ]) - // remove the big test logs zip file, store the nightlies url in test_logs.html - sh '''#!/bin/bash -e - if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then - echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" - rm -rf "${OUTPUT_DIR}/test_logs.zip" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" - else - echo "No test_logs.zip, skipping" - fi -''' - // Has to be relative to WORKSPACE. - archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" - archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" - publishHTML target: [ - allowMissing : true, - keepAll : true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE. - reportDir : "${env.OUTPUT_DIR_RELATIVE}", - reportFiles : 'console-report.html', - reportName : 'JDK7 Nightly Build Report' - ] - } - } - } - stage ('yetus jdk8 hadoop2 checks') { - agent { - node { - label 'hbase' - } + branch 'branch-2*' } environment { BASEDIR = "${env.WORKSPACE}/component" @@ -519,11 +410,6 @@ pipeline { label 'hbase' } } - when { - not { - branch 'branch-1*' - } - } environment { BASEDIR = "${env.WORKSPACE}/component" TESTS = "${env.DEEP_CHECKS}" @@ -633,11 +519,6 @@ pipeline { label 'hbase' } } - when { - not { - branch 'branch-1*' - } - } environment { BASEDIR = "${env.WORKSPACE}/component" TESTS = "${env.DEEP_CHECKS}" @@ -814,7 +695,7 @@ pipeline { ''' unstash 'hadoop-2' sh '''#!/bin/bash -xe - if [[ "${BRANCH}" = branch-2* ]] || [[ "${BRANCH}" = branch-1* ]]; then + if [[ "${BRANCH}" = branch-2* ]]; then echo "Attempting to use run an instance on top of Hadoop 2." artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1) tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2" @@ -838,44 +719,40 @@ pipeline { ''' unstash 'hadoop-3' sh '''#!/bin/bash -e - if [[ "${BRANCH}" = branch-1* ]]; then - echo "Skipping to run against Hadoop 3 for branch ${BRANCH}" - else - echo "Attempting to use run an instance on top of Hadoop 3." - artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1) - tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" - if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ - --single-process \ - --working-dir output-integration/hadoop-3 \ - --hbase-client-install hbase-client \ - hbase-install \ - hadoop-3/bin/hadoop \ - hadoop-3/share/hadoop/yarn/timelineservice \ - hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-3/bin/mapred \ - >output-integration/hadoop-3.log 2>&1 ; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile - exit 2 - fi - echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program." - if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ - --single-process \ - --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \ - --working-dir output-integration/hadoop-3-shaded \ - --hbase-client-install hbase-client \ - hbase-install \ - hadoop-3/bin/hadoop \ - hadoop-3/share/hadoop/yarn/timelineservice \ - hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-3/bin/mapred \ - >output-integration/hadoop-3-shaded.log 2>&1 ; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile - exit 2 - fi - echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile + echo "Attempting to use run an instance on top of Hadoop 3." + artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1) + tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" + if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ + --single-process \ + --working-dir output-integration/hadoop-3 \ + --hbase-client-install hbase-client \ + hbase-install \ + hadoop-3/bin/hadoop \ + hadoop-3/share/hadoop/yarn/timelineservice \ + hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + hadoop-3/bin/mapred \ + >output-integration/hadoop-3.log 2>&1 ; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile + exit 2 fi + echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program." + if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ + --single-process \ + --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \ + --working-dir output-integration/hadoop-3-shaded \ + --hbase-client-install hbase-client \ + hbase-install \ + hadoop-3/bin/hadoop \ + hadoop-3/share/hadoop/yarn/timelineservice \ + hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + hadoop-3/bin/mapred \ + >output-integration/hadoop-3-shaded.log 2>&1 ; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile + exit 2 + fi + echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile ''' } post { @@ -916,14 +793,12 @@ pipeline { script { try { unstash 'general-result' - unstash 'jdk7-result' unstash 'jdk8-hadoop2-result' unstash 'jdk8-hadoop3-result' unstash 'jdk11-hadoop3-result' unstash 'srctarball-result' sh "printenv" def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile", - "${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile", @@ -948,8 +823,14 @@ pipeline { echo "[INFO] Comment:" echo comment echo "" - echo "[INFO] There are ${currentBuild.changeSets.size()} change sets." - getJirasToComment(currentBuild).each { currentIssue -> + echo "[DEBUG] checking to see if feature branch" + def jiras = getJirasToComment(env.BRANCH_NAME, []) + if (jiras.isEmpty()) { + echo "[DEBUG] non-feature branch, checking change messages for jira keys." + echo "[INFO] There are ${currentBuild.changeSets.size()} change sets." + jiras = getJirasToCommentFromChangesets(currentBuild) + } + jiras.each { currentIssue -> jiraComment issueKey: currentIssue, body: comment } } catch (Exception exception) { @@ -962,7 +843,7 @@ pipeline { } import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper @NonCPS -List getJirasToComment(RunWrapper thisBuild) { +List getJirasToCommentFromChangesets(RunWrapper thisBuild) { def seenJiras = [] thisBuild.changeSets.each { cs -> cs.getItems().each { change -> @@ -972,16 +853,21 @@ List getJirasToComment(RunWrapper thisBuild) { echo " ${change.commitId}" echo " ${change.author}" echo "" - msg.eachMatch("HBASE-[0-9]+") { currentIssue -> - echo "[DEBUG] found jira key: ${currentIssue}" - if (currentIssue in seenJiras) { - echo "[DEBUG] already commented on ${currentIssue}." - } else { - echo "[INFO] commenting on ${currentIssue}." - seenJiras << currentIssue - } - } + seenJiras = getJirasToComment(msg, seenJiras) } } return seenJiras } +@NonCPS +List getJirasToComment(CharSequence source, List seen) { + source.eachMatch("HBASE-[0-9]+") { currentIssue -> + echo "[DEBUG] found jira key: ${currentIssue}" + if (currentIssue in seen) { + echo "[DEBUG] already commented on ${currentIssue}." + } else { + echo "[INFO] commenting on ${currentIssue}." + seen << currentIssue + } + } + return seen +} diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index e286a0ce749..68a75a9f34a 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -80,9 +80,9 @@ function personality_globals # TODO use PATCH_BRANCH to select jdk versions to use. # Yetus 0.7.0 enforces limits. Default proclimit is 1000. - # Up it. See HBASE-19902 for how we arrived at this number. + # Up it. See HBASE-25081 for how we arrived at this number. #shellcheck disable=SC2034 - PROC_LIMIT=12500 + PROC_LIMIT=30000 # Set docker container to run with 20g. Default is 4g in yetus. # See HBASE-19902 for how we arrived at 20g. @@ -145,8 +145,6 @@ function personality_modules local repostatus=$1 local testtype=$2 local extra="" - local branch1jdk8=() - local jdk8module="" local MODULES=("${CHANGED_MODULES[@]}") yetus_info "Personality: ${repostatus} ${testtype}" @@ -173,10 +171,6 @@ function personality_modules tmpdir=$(realpath target) extra="${extra} -Djava.io.tmpdir=${tmpdir} -DHBasePatchProcess" - if [[ "${PATCH_BRANCH}" = branch-1* ]]; then - extra="${extra} -Dhttps.protocols=TLSv1.2" - fi - # If we have HADOOP_PROFILE specified and we're on branch-2.x, pass along # the hadoop.profile system property. Ensures that Hadoop2 and Hadoop3 # logic is not both activated within Maven. @@ -207,21 +201,6 @@ function personality_modules return fi - # This list should include any modules that require jdk8. Maven should be configured to only - # include them when a proper JDK is in use, but that doesn' work if we specifically ask for the - # module to build as yetus does if something changes in the module. Rather than try to - # figure out what jdk is in use so we can duplicate the module activation logic, just - # build at the top level if anything changes in one of these modules and let maven sort it out. - branch1jdk8=(hbase-error-prone hbase-tinylfu-blockcache) - if [[ "${PATCH_BRANCH}" = branch-1* ]]; then - for jdk8module in "${branch1jdk8[@]}"; do - if [[ "${MODULES[*]}" =~ ${jdk8module} ]]; then - MODULES=(.) - break - fi - done - fi - if [[ ${testtype} == spotbugs ]]; then # Run spotbugs on each module individually to diff pre-patch and post-patch results and # report new warnings for changed modules only. @@ -241,8 +220,7 @@ function personality_modules return fi - if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] && - [[ "${PATCH_BRANCH}" != branch-1* ]] ; then + if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]]; then extra="${extra} -PerrorProne" fi @@ -445,11 +423,7 @@ function refguide_rebuild return 1 fi - if [[ "${PATCH_BRANCH}" = branch-1* ]]; then - pdf_output="book.pdf" - else - pdf_output="apache_hbase_reference_guide.pdf" - fi + pdf_output="apache_hbase_reference_guide.pdf" if [[ ! -f "${PATCH_DIR}/${repostatus}-site/${pdf_output}" ]]; then add_vote_table -1 refguide "${repostatus} failed to produce the pdf version of the reference guide." @@ -601,76 +575,22 @@ function hadoopcheck_rebuild # All supported Hadoop versions that we want to test the compilation with # See the Hadoop section on prereqs in the HBase Reference Guide - if [[ "${PATCH_BRANCH}" = branch-1.3 ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-1.3 rules." + if [[ "${PATCH_BRANCH}" = branch-2.* ]]; then + yetus_info "Setting Hadoop 2 versions to test based on branch-2.4+ rules." if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.7" + hbase_hadoop2_versions="2.10.1" else - hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7" - fi - elif [[ "${PATCH_BRANCH}" = branch-1.4 ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-1.4 rules." - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.7.7" - else - hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7" - fi - elif [[ "${PATCH_BRANCH}" = branch-1 ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-1 rules." - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.10.0" - else - hbase_hadoop2_versions="2.10.0" - fi - elif [[ "${PATCH_BRANCH}" = branch-2.0 ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-2.0 rules." - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.6.5 2.7.7 2.8.5" - else - hbase_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5" - fi - elif [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-2.1 rules." - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.7.7 2.8.5" - else - hbase_hadoop2_versions="2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.8.2 2.8.3 2.8.4 2.8.5" - fi - elif [[ "${PATCH_BRANCH}" = branch-2.2 ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-2.2 rules." - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0" - else - hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0" - fi - elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then - yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules." - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop2_versions="2.10.0" - else - hbase_hadoop2_versions="2.10.0" + hbase_hadoop2_versions="2.10.0 2.10.1" fi else yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more." hbase_hadoop2_versions="" fi - if [[ "${PATCH_BRANCH}" = branch-1* ]]; then - yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules." - hbase_hadoop3_versions="" - elif [[ "${PATCH_BRANCH}" = branch-2.0 ]] || [[ "${PATCH_BRANCH}" = branch-2.1 ]]; then - yetus_info "Setting Hadoop 3 versions to test based on branch-2.0/branch-2.1 rules" - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop3_versions="3.0.3 3.1.2" - else - hbase_hadoop3_versions="3.0.3 3.1.1 3.1.2" - fi + yetus_info "Setting Hadoop 3 versions to test based on branch-2.4+/master/feature branch rules" + if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then + hbase_hadoop3_versions="3.1.2 3.2.2 3.3.1" else - yetus_info "Setting Hadoop 3 versions to test based on branch-2.2+/master/feature branch rules" - if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop3_versions="3.1.2 3.2.1" - else - hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1" - fi + hbase_hadoop3_versions="3.1.1 3.1.2 3.2.0 3.2.1 3.2.2 3.3.0 3.3.1" fi export MAVEN_OPTS="${MAVEN_OPTS}"