diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub new file mode 100644 index 00000000000..f63e4b0d223 --- /dev/null +++ b/dev-support/Jenkinsfile_GitHub @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pipeline { + + agent { + label 'Hadoop || ubuntu' + } + + options { + buildDiscarder(logRotator(numToKeepStr: '30')) + timeout (time: 7, unit: 'HOURS') + timestamps() + checkoutToSubdirectory('src') + } + + environment { + SOURCEDIR = 'src' + // will also need to change notification section below + PATCHDIR = 'out' + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" + YETUS='yetus' + // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' + YETUS_VERSION='rel/0.9.0' + } + + parameters { + string(name: 'JIRA_ISSUE_KEY', + defaultValue: '', + description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HBASE-1234') + } + + stages { + stage ('install yetus') { + steps { + dir("${WORKSPACE}/${YETUS}") { + checkout([ + $class: 'GitSCM', + branches: [[name: "${env.YETUS_VERSION}"]], + userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]] + ) + } + } + } + + stage ('precommit-run') { + steps { + withCredentials( + [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_PASSWORD', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')]) { + sh '''#!/usr/bin/env bash + set -e + TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" + # this must be clean for every run + if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then + rm -rf "${WORKSPACE}/${PATCHDIR}" + fi + mkdir -p "${WORKSPACE}/${PATCHDIR}" + + ## Checking on H* machine nonsense + echo "JAVA_HOME: ${JAVA_HOME}" + ls -l "${JAVA_HOME}" || true + echo "MAVEN_HOME: ${MAVEN_HOME}" + echo "maven version:" + mvn --offline --version || true + echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/" + mkdir "${PATCHDIR}/machine" + cat /proc/cpuinfo >"${PATCHDIR}/machine/cpuinfo" 2>&1 || true + cat /proc/meminfo >"${PATCHDIR}/machine/meminfo" 2>&1 || true + cat /proc/diskstats >"${PATCHDIR}/machine/diskstats" 2>&1 || true + cat /sys/block/sda/stat >"${PATCHDIR}/machine/sys-block-sda-stat" 2>&1 || true + df -h >"${PATCHDIR}/machine/df-h" 2>&1 || true + ps -Awwf >"${PATCHDIR}/machine/ps-Awwf" 2>&1 || true + ifconfig -a >"${PATCHDIR}/machine/ifconfig-a" 2>&1 || true + lsblk -ta >"${PATCHDIR}/machine/lsblk-ta" 2>&1 || true + lsblk -fa >"${PATCHDIR}/machine/lsblk-fa" 2>&1 || true + cat /proc/loadavg >"${output}/loadavg" 2>&1 || true + ulimit -a >"${PATCHDIR}/machine/ulimit-a" 2>&1 || true + ## /H* + + # if given a JIRA issue, process it. If CHANGE_URL is set + # (e.g., Github Branch Source plugin), process it. + # otherwise exit, because we don't want HBase to do a + # full build. We wouldn't normally do this check for smaller + # projects. :) + if [[ -n "${JIRA_ISSUE_KEY}" ]]; then + YETUS_ARGS+=("${JIRA_ISSUE_KEY}") + elif [[ -z "${CHANGE_URL}" ]]; then + echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html" + exit 0 + fi + YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") + # where the source is located + YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") + # our project defaults come from a personality file + # which will get loaded automatically by setting the project name + YETUS_ARGS+=("--project=hbase") + # lots of different output formats + YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") + YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") + YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") + # enable writing back to Github + YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}") + YETUS_ARGS+=(--github-user=${GITHUB_USER}) + # enable writing back to ASF JIRA + YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}") + YETUS_ARGS+=(--jira-user="${JIRA_USER}") + # auto-kill any surefire stragglers during unit test runs + YETUS_ARGS+=("--reapermode=kill") + YETUS_ARGS+=("--multijdktests=compile") + # set relatively high limits for ASF machines + # changing these to higher values may cause problems + # with other jobs on systemd-enabled machines + YETUS_ARGS+=("--proclimit=10000") + YETUS_ARGS+=("--dockermemlimit=20g") + # -1 findbugs issues that show up prior to the patch being applied + YETUS_ARGS+=("--findbugs-strict-precheck") + # rsync these files back into the archive dir + YETUS_ARGS+=("--archive-list=rat.txt") + # URL for user-side presentation in reports and such to our artifacts + # (needs to match the archive bits below) + YETUS_ARGS+=("--build-url-artifacts=artifact/out") + # plugins to enable + YETUS_ARGS+=("--plugins=all") + # don't let these tests cause -1s because we aren't really paying that + # much attention to them + YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests") + # run in docker mode and specifically point to our + # Dockerfile since we don't want to use the auto-pulled version. + YETUS_ARGS+=("--docker") + YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64") + YETUS_ARGS+=("--findbugs-home=/usr") + YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*") + YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*") + YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh") + YETUS_ARGS+=("--quick-hadoopcheck") + # effectively treat dev-support as a custom maven module + YETUS_ARGS+=("--skip-dir=dev-support") + # help keep the ASF boxes clean + YETUS_ARGS+=("--sentinel") + "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" + ''' + } + } + } + + } + + post { + always { + script { + // Yetus output + archiveArtifacts "${env.PATCHDIR}/**" + // Publish the HTML report so that it can be looked at + // Has to be relative to WORKSPACE. + publishHTML (target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: "${env.PATCHDIR}", + reportFiles: 'report.html', + reportName: 'Yetus Report' + ]) + // Publish JUnit results + try { + junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml" + } catch(e) { + echo 'junit processing: ' + e.toString() + } + } + } + + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh ''' + # See YETUS-764 + if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then + echo "test-patch process appears to still be running: killing" + kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true + sleep 10 + fi + if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then + echo "test-patch container appears to still be running: killing" + docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true + fi + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + deleteDir() + } + } + } +}