HBASE-20387 turn flaky test tracking infra into per-branch pipeline.

* gather up all the flaky test stuff into a directory
* create Jenkins Pipeline DSL for the report generation and the flaky re-testing
* have the nightly per-branch job consume the results of flaky reporting

Signed-off-by: Mike Drob <mdrob@apache.org>
This commit is contained in:
Sean Busbey 2018-08-10 11:28:10 -05:00
parent 1dbd6fa993
commit f9793fafb7
8 changed files with 141 additions and 9 deletions

View File

@ -17,6 +17,9 @@
#
# This Dockerfile is to setup environment for dev-support scripts which require
# dependencies outside of what Apache Jenkins machines may have.
#
# Specifically, it's used for the flaky test reporting job defined in
# dev-support/flaky-tests/flaky-reporting.Jenkinsfile
FROM ubuntu:14.04
ADD . /hbase/dev-support

View File

@ -47,11 +47,7 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
// Flaky urls for different branches. Replace '-' and '.' in branch name by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
// EXCLUDE_TESTS_URL_master = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
// EXCLUDE_TESTS_URL_branch_2 = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests-branch2.0/lastSuccessfulBuild/artifact/excludes/'
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes"
}
parameters {
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.

View File

@ -0,0 +1,66 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
pipeline {
agent {
node {
label 'Hadoop'
}
}
triggers {
cron('@daily')
}
options {
buildDiscarder(logRotator(numToKeepStr: '100'))
timeout (time: 15, unit: 'MINUTES')
timestamps()
}
parameters {
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
}
stages {
stage ('build flaky report') {
steps {
sh '''#!/usr/bin/env bash
set -e
if [ "${DEBUG}" = "true" ]; then
set -x
fi
declare -a flaky_args
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 5)
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 40)
docker build -t hbase-dev-support dev-support
docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
'''
}
}
}
post {
always {
// Has to be relative to WORKSPACE.
archive "includes,excludes,dashboard.html"
publishHTML target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: ".",
reportFiles: 'dashboard.html',
reportName: 'Flaky Test Report'
]
}
}
}

View File

@ -0,0 +1,71 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
pipeline {
agent {
node {
label 'Hadoop'
}
}
triggers {
cron('@hourly')
}
options {
// this should roughly match how long we tell the flaky dashboard to look at
buildDiscarder(logRotator(numToKeepStr: '80'))
timeout (time: 2, unit: 'HOURS')
timestamps()
}
parameters {
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
}
tools {
// this should match what the yetus nightly job for the branch will use
maven 'Maven (latest)'
jdk "JDK 1.8 (latest)"
}
stages {
stage ('run flaky tests') {
steps {
sh '''#!/usr/bin/env bash
set -e
declare -a curl_args=(--fail)
declare -a mvn_args=(--batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dmaven.repo.local="${WORKSPACE}/local-repository")
if [ "${DEBUG}" = "true" ]; then
curl_args=("${curl_args[@]}" -v)
mvn_args=("${mvn_args[@]}" -X)
set -x
fi
ulimit -a
rm -rf local-repository/org/apache/hbase
curl "${curl_args[@]}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes"
if [ -s includes.txt ]; then
mvn clean package "${mvn_args[@]}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dsurefire.firstPartForkCount=3 -Dsurefire.secondPartForkCount=3
else
echo "set of flaky tests is currently empty."
fi
'''
}
}
}
post {
always {
junit testResults: "**/surefire-reports/*.xml", allowEmptyResults: true
// TODO compress these logs
archive 'includes.txt,**/surefire-reports/*,**/test-data/*'
}
}
}

View File

@ -71,10 +71,6 @@ YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}")
# Currently, flaky list is calculated only for master branch.
UNDERSCORED_BRANCH_NAME=$(echo ${BRANCH_NAME} | tr '.-' '_')
EXCLUDE_TESTS_URL=$(eval echo "\$EXCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
INCLUDE_TESTS_URL=$(eval echo "\$INCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
fi