hbase/dev-support/Jenkinsfile_GitHub

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

pipeline {

    agent {
        label 'Hadoop'
    }

    options {
        // N.B. this is per-branch, which means per PR
        disableConcurrentBuilds()
        buildDiscarder(logRotator(numToKeepStr: '15'))
        timeout (time: 7, unit: 'HOURS')
        timestamps()
        checkoutToSubdirectory('src')
    }

    environment {
        SOURCEDIR = 'src'
        // will also need to change notification section below
        PATCHDIR = 'out'
        DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
        YETUS='yetus'
        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
        YETUS_VERSION='rel/0.11.0'
    }

    parameters {
        booleanParam(name: 'DEBUG',
               defaultValue: false,
               description: 'Print extra outputs for debugging the jenkins job and yetus')
    }

    stages {
        stage ('install yetus') {
            steps {
                dir("${WORKSPACE}/${YETUS}") {
                    checkout([
                        $class: 'GitSCM',
                        branches: [[name: "${env.YETUS_VERSION}"]],
                        userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
                    )
                }
            }
        }

        stage ('precommit-run') {
            steps {
                withCredentials(
                    [usernamePassword(credentialsId: 'apache-hbase-at-github.com',
                                  passwordVariable: 'GITHUB_PASSWORD',
                                  usernameVariable: 'GITHUB_USER'),
                    usernamePassword(credentialsId: 'hbaseqa-at-asf-jira',
                                        passwordVariable: 'JIRA_PASSWORD',
                                        usernameVariable: 'JIRA_USER')]) {
                        sh '''#!/usr/bin/env bash
                        set -e
                        TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
                        # this must be clean for every run
                        if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
                          rm -rf "${WORKSPACE}/${PATCHDIR}"
                        fi
                        mkdir -p "${WORKSPACE}/${PATCHDIR}"

                        ## Checking on H* machine nonsense
                        echo "JAVA_HOME: ${JAVA_HOME}"
                        ls -l "${JAVA_HOME}" || true
                        echo "MAVEN_HOME: ${MAVEN_HOME}"
                        echo "maven version:"
                        mvn --offline --version  || true
                        echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/"
                        mkdir "${PATCHDIR}/machine"
                        cat /proc/cpuinfo >"${PATCHDIR}/machine/cpuinfo" 2>&1 || true
                        cat /proc/meminfo >"${PATCHDIR}/machine/meminfo" 2>&1 || true
                        cat /proc/diskstats >"${PATCHDIR}/machine/diskstats" 2>&1 || true
                        cat /sys/block/sda/stat >"${PATCHDIR}/machine/sys-block-sda-stat" 2>&1 || true
                        df -h >"${PATCHDIR}/machine/df-h" 2>&1 || true
                        ps -Awwf >"${PATCHDIR}/machine/ps-Awwf" 2>&1 || true
                        ifconfig -a >"${PATCHDIR}/machine/ifconfig-a" 2>&1 || true
                        lsblk -ta >"${PATCHDIR}/machine/lsblk-ta" 2>&1 || true
                        lsblk -fa >"${PATCHDIR}/machine/lsblk-fa" 2>&1 || true
                        cat /proc/loadavg >"${output}/loadavg" 2>&1 || true
                        ulimit -a >"${PATCHDIR}/machine/ulimit-a" 2>&1 || true
                        ## /H*

                        # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
                        # Otherwise exit, because we don't want HBase to do a
                        # full build.  We wouldn't normally do this check for smaller
                        # projects. :)
                        if [[ -z "${CHANGE_URL}" ]]; then
                            echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
                            exit 0
                        fi
                        # enable debug output for yetus
                        if [[ "true" = "${DEBUG}" ]]; then
                            YETUS_ARGS+=("--debug")
                        fi
                        # If we're doing docker, make sure we don't accidentally pollute the image with a host java path
                        if [ -n "${JAVA_HOME}" ]; then
                          unset JAVA_HOME
                        fi
                        YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
                        # where the source is located
                        YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
                        # our project defaults come from a personality file
                        # which will get loaded automatically by setting the project name
                        YETUS_ARGS+=("--project=hbase")
                        # lots of different output formats
                        YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
                        YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
                        YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
                        # enable writing back to Github
                        YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
                        YETUS_ARGS+=(--github-user=${GITHUB_USER})
                        # enable writing back to ASF JIRA
                        YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
                        YETUS_ARGS+=(--jira-user="${JIRA_USER}")
                        # auto-kill any surefire stragglers during unit test runs
                        YETUS_ARGS+=("--reapermode=kill")
                        YETUS_ARGS+=("--multijdktests=compile")
                        # set relatively high limits for ASF machines
                        # changing these to higher values may cause problems
                        # with other jobs on systemd-enabled machines
                        YETUS_ARGS+=("--proclimit=10000")
                        YETUS_ARGS+=("--dockermemlimit=20g")
                        # -1 findbugs issues that show up prior to the patch being applied
                        YETUS_ARGS+=("--findbugs-strict-precheck")
                        # rsync these files back into the archive dir
                        YETUS_ARGS+=("--archive-list=rat.txt")
                        # URL for user-side presentation in reports and such to our artifacts
                        # (needs to match the archive bits below)
                        YETUS_ARGS+=("--build-url-artifacts=artifact/out")
                        # plugins to enable
                        YETUS_ARGS+=("--plugins=all")
                        # don't let these tests cause -1s because we aren't really paying that
                        # much attention to them
                        YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests")
                        # run in docker mode and specifically point to our
                        # Dockerfile since we don't want to use the auto-pulled version.
                        YETUS_ARGS+=("--docker")
                        YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
                        YETUS_ARGS+=("--mvn-custom-repos")
                        YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64")
                        YETUS_ARGS+=("--findbugs-home=/usr")
                        YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
                        YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
                        YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
                        YETUS_ARGS+=("--quick-hadoopcheck")
                        YETUS_ARGS+=("--skip-errorprone")
                        # effectively treat dev-support as a custom maven module
                        YETUS_ARGS+=("--skip-dirs=dev-support")
                        # help keep the ASF boxes clean
                        YETUS_ARGS+=("--sentinel")
                        # use emoji vote so it is easier to find the broken line
                        YETUS_ARGS+=("--github-use-emoji-vote")
                        "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
                        '''
                }
            }
        }

    }

    post {
        always {
          script {
            // Yetus output
            archiveArtifacts "${env.PATCHDIR}/**"
            // Publish the HTML report so that it can be looked at
            // Has to be relative to WORKSPACE.
            publishHTML (target: [
                          allowMissing: true,
                          keepAll: true,
                          alwaysLinkToLastBuild: true,
                          // Has to be relative to WORKSPACE
                          reportDir: "${env.PATCHDIR}",
                          reportFiles: 'report.html',
                          reportName: 'Yetus Report'
            ])
            // Publish JUnit results
            try {
                junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
            } catch(e) {
                echo 'junit processing: ' + e.toString()
            }
          }
        }

        // Jenkins pipeline jobs fill slaves on PRs without this :(
        cleanup() {
            script {
                sh '''
                    # See YETUS-764
                    if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
                      echo "test-patch process appears to still be running: killing"
                      kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
                      sleep 10
                    fi
                    if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
                      echo "test-patch container appears to still be running: killing"
                      docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
                    fi
                    # See HADOOP-13951
                    chmod -R u+rxw "${WORKSPACE}"
                    '''
                deleteDir()
            }
        }
    }
}