a4272a9f08
Jenkins fails the whole build immediately if any stage fails. Hadoop2 tests run before Hadoop3 tests. So Hadoop3 tests will run only if hadoop2 tests pass.
360 lines
14 KiB
Groovy
360 lines
14 KiB
Groovy
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
pipeline {
|
|
agent {
|
|
node {
|
|
label 'Hadoop'
|
|
}
|
|
}
|
|
triggers {
|
|
cron('@daily')
|
|
}
|
|
options {
|
|
buildDiscarder(logRotator(numToKeepStr: '30'))
|
|
timeout (time: 6, unit: 'HOURS')
|
|
timestamps()
|
|
skipDefaultCheckout()
|
|
}
|
|
environment {
|
|
TOOLS = "${env.WORKSPACE}/tools"
|
|
// where we check out to across stages
|
|
BASEDIR = "${env.WORKSPACE}/component"
|
|
YETUS_RELEASE = '0.6.0'
|
|
PROJECT = 'hbase'
|
|
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
|
|
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
|
|
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
|
|
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
|
|
// output from surefire; sadly the archive function in yetus only works on file names.
|
|
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
|
|
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
|
|
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
|
|
BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
|
|
// Only used for master branch.
|
|
EXCLUDE_TESTS_URL = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
|
|
}
|
|
parameters {
|
|
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
|
|
|
|
Should only be used manually when e.g. there is some non-work-aroundable issue in yetus we are checking a fix for.''')
|
|
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
|
|
}
|
|
stages {
|
|
stage ('scm checkout') {
|
|
steps {
|
|
dir('component') {
|
|
checkout scm
|
|
}
|
|
}
|
|
}
|
|
stage ('yetus install') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
echo "Ensure we have a copy of Apache Yetus."
|
|
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
|
|
YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
|
|
echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
|
|
if [ ! -d "${YETUS_DIR}" ]; then
|
|
echo "New download of Apache Yetus version ${YETUS_RELEASE}."
|
|
rm -rf "${WORKSPACE}/.gpg"
|
|
mkdir -p "${WORKSPACE}/.gpg"
|
|
chmod -R 700 "${WORKSPACE}/.gpg"
|
|
|
|
echo "install yetus project KEYS"
|
|
curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS
|
|
gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
|
|
|
|
echo "download yetus release ${YETUS_RELEASE}"
|
|
curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
|
|
curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
|
|
echo "verifying yetus release"
|
|
gpg --homedir "${WORKSPACE}/.gpg" --verify "yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
|
|
mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
|
|
else
|
|
echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
|
|
fi
|
|
else
|
|
YETUS_DIR="${WORKSPACE}/yetus-git"
|
|
rm -rf "${YETUS_DIR}"
|
|
echo "downloading from github"
|
|
curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz
|
|
fi
|
|
if [ ! -d "${YETUS_DIR}" ]; then
|
|
echo "unpacking yetus into '${YETUS_DIR}'"
|
|
mkdir -p "${YETUS_DIR}"
|
|
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
|
|
fi
|
|
'''
|
|
dir ("${env.TOOLS}") {
|
|
sh """#!/usr/bin/env bash
|
|
echo "Downloading Project personality."
|
|
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
|
|
"""
|
|
}
|
|
stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
|
|
}
|
|
}
|
|
stage ('yetus general check') {
|
|
environment {
|
|
// TODO does hadoopcheck need to be jdk specific?
|
|
// Should be things that work with multijdk
|
|
TESTS = 'all,-unit,-findbugs'
|
|
// on branches that don't support jdk7, this will already be JAVA_HOME, so we'll end up not
|
|
// doing multijdk there.
|
|
MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
|
|
OUTPUT_DIR_RELATIVE = "output-general"
|
|
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
|
|
}
|
|
steps {
|
|
unstash 'yetus'
|
|
sh '''#!/usr/bin/env bash
|
|
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
|
|
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
|
|
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
|
|
'''
|
|
// TODO should this be a download from master, similar to how the personality is?
|
|
sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
|
|
}
|
|
post {
|
|
always {
|
|
// Has to be relative to WORKSPACE.
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/*"
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
|
|
publishHTML target: [
|
|
allowMissing: true,
|
|
keepAll: true,
|
|
alwaysLinkToLastBuild: true,
|
|
// Has to be relative to WORKSPACE
|
|
reportDir: "${env.OUTPUT_DIR_RELATIVE}",
|
|
reportFiles: 'console-report.html',
|
|
reportName: 'General Nightly Build Report'
|
|
]
|
|
}
|
|
}
|
|
}
|
|
stage ('yetus jdk7 checks') {
|
|
when {
|
|
branch 'branch-1*'
|
|
}
|
|
environment {
|
|
TESTS = 'mvninstall,compile,javac,unit,htmlout'
|
|
OUTPUT_DIR_RELATIVE = "output-jdk7"
|
|
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
|
|
// On branches where we do jdk7 checks, jdk7 will be JAVA_HOME already.
|
|
}
|
|
steps {
|
|
unstash 'yetus'
|
|
sh '''#!/usr/bin/env bash
|
|
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
|
|
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
|
|
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
|
|
'''
|
|
sh """#!/usr/bin/env bash
|
|
# for branch-1.1 we don't do jdk8 findbugs, so do it here
|
|
if [ "${env.BRANCH_NAME}" == "branch-1.1" ]; then
|
|
TESTS+=",findbugs"
|
|
fi
|
|
"${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
|
|
"""
|
|
}
|
|
post {
|
|
always {
|
|
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
|
|
// zip surefire reports.
|
|
sh '''#!/bin/bash -e
|
|
if [ -d "${OUTPUT_DIR}/archiver" ]; then
|
|
count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
|
|
if [[ 0 -ne ${count} ]]; then
|
|
echo "zipping ${count} archived files"
|
|
zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
|
|
else
|
|
echo "No archived files, skipping compressing."
|
|
fi
|
|
else
|
|
echo "No archiver directory, skipping compressing."
|
|
fi
|
|
'''
|
|
// Has to be relative to WORKSPACE.
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/*"
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
|
|
publishHTML target: [
|
|
allowMissing : true,
|
|
keepAll : true,
|
|
alwaysLinkToLastBuild: true,
|
|
// Has to be relative to WORKSPACE.
|
|
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
|
|
reportFiles : 'console-report.html',
|
|
reportName : 'JDK7 Nightly Build Report'
|
|
]
|
|
}
|
|
}
|
|
}
|
|
stage ('yetus jdk8 hadoop2 checks') {
|
|
when {
|
|
not {
|
|
branch 'branch-1.1*'
|
|
}
|
|
}
|
|
environment {
|
|
TESTS = 'mvninstall,compile,javac,unit,htmlout'
|
|
OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop2"
|
|
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
|
|
// This isn't strictly needed on branches that only support jdk8, but doesn't hurt
|
|
// and is needed on branches that do both jdk7 and jdk8
|
|
SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
|
|
}
|
|
steps {
|
|
unstash 'yetus'
|
|
sh '''#!/usr/bin/env bash
|
|
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
|
|
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
|
|
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
|
|
'''
|
|
sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
|
|
}
|
|
post {
|
|
always {
|
|
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
|
|
// zip surefire reports.
|
|
sh '''#!/bin/bash -e
|
|
if [ -d "${OUTPUT_DIR}/archiver" ]; then
|
|
count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
|
|
if [[ 0 -ne ${count} ]]; then
|
|
echo "zipping ${count} archived files"
|
|
zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
|
|
else
|
|
echo "No archived files, skipping compressing."
|
|
fi
|
|
else
|
|
echo "No archiver directory, skipping compressing."
|
|
fi
|
|
'''
|
|
// Has to be relative to WORKSPACE.
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/*"
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
|
|
publishHTML target: [
|
|
allowMissing : true,
|
|
keepAll : true,
|
|
alwaysLinkToLastBuild: true,
|
|
// Has to be relative to WORKSPACE.
|
|
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
|
|
reportFiles : 'console-report.html',
|
|
reportName : 'JDK8 Nightly Build Report (Hadoop2)'
|
|
]
|
|
}
|
|
}
|
|
}
|
|
stage ('yetus jdk8 hadoop3 checks') {
|
|
when {
|
|
not {
|
|
branch 'branch-1*'
|
|
}
|
|
}
|
|
environment {
|
|
// Failure in any stage fails the build and consecutive stages are not built.
|
|
// Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3
|
|
// tests.
|
|
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
|
|
OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop3"
|
|
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
|
|
// This isn't strictly needed on branches that only support jdk8, but doesn't hurt
|
|
// and is needed on branches that do both jdk7 and jdk8
|
|
SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
|
|
// Activates hadoop 3.0 profile in maven runs.
|
|
HADOOP_PROFILE = '3.0'
|
|
}
|
|
steps {
|
|
unstash 'yetus'
|
|
sh '''#!/usr/bin/env bash
|
|
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
|
|
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
|
|
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
|
|
'''
|
|
sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
|
|
}
|
|
post {
|
|
always {
|
|
// Not sure how two junit test reports will work. Disabling this for now.
|
|
// junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
|
|
// zip surefire reports.
|
|
sh '''#!/bin/bash -e
|
|
if [ -d "${OUTPUT_DIR}/archiver" ]; then
|
|
count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
|
|
if [[ 0 -ne ${count} ]]; then
|
|
echo "zipping ${count} archived files"
|
|
zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
|
|
else
|
|
echo "No archived files, skipping compressing."
|
|
fi
|
|
else
|
|
echo "No archiver directory, skipping compressing."
|
|
fi
|
|
'''
|
|
// Has to be relative to WORKSPACE.
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/*"
|
|
archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
|
|
publishHTML target: [
|
|
allowMissing : true,
|
|
keepAll : true,
|
|
alwaysLinkToLastBuild: true,
|
|
// Has to be relative to WORKSPACE.
|
|
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
|
|
reportFiles : 'console-report.html',
|
|
reportName : 'JDK8 Nightly Build Report (Hadoop3)'
|
|
]
|
|
}
|
|
}
|
|
}
|
|
// This is meant to mimic what a release manager will do to create RCs.
|
|
// See http://hbase.apache.org/book.html#maven.release
|
|
stage ('create source tarball') {
|
|
tools {
|
|
maven 'Maven (latest)'
|
|
// this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
|
|
jdk "JDK 1.8 (latest)"
|
|
}
|
|
steps {
|
|
sh '''#!/bin/bash -e
|
|
echo "Setting up directories"
|
|
rm -rf "output-srctarball" && mkdir "output-srctarball"
|
|
rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
|
|
rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
|
|
rm -rf ".m2-for-src" && mkdir ".m2-for-src"
|
|
'''
|
|
sh '''#!/usr/bin/env bash
|
|
rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
|
|
"${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
|
|
'''
|
|
sh """#!/bin/bash -e
|
|
${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh \
|
|
--intermediate-file-dir output-srctarball \
|
|
--unpack-temp-dir unpacked_src_tarball \
|
|
--maven-m2-initial .m2-for-repo \
|
|
--maven-m2-src-build .m2-for-src \
|
|
--clean-source-checkout \
|
|
${env.BASEDIR}
|
|
"""
|
|
}
|
|
post {
|
|
always {
|
|
archive 'output-srctarball/*'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|