HBASE-20387 turn flaky test tracking infra into per-branch pipeline.
* gather up all the flaky test stuff into a directory * create Jenkins Pipeline DSL for the report generation and the flaky re-testing * have the nightly per-branch job consume the results of flaky reporting Signed-off-by: Mike Drob <mdrob@apache.org>
This commit is contained in:
parent
1dbd6fa993
commit
f9793fafb7
|
@ -17,6 +17,9 @@
|
|||
#
|
||||
# This Dockerfile is to setup environment for dev-support scripts which require
|
||||
# dependencies outside of what Apache Jenkins machines may have.
|
||||
#
|
||||
# Specifically, it's used for the flaky test reporting job defined in
|
||||
# dev-support/flaky-tests/flaky-reporting.Jenkinsfile
|
||||
FROM ubuntu:14.04
|
||||
|
||||
ADD . /hbase/dev-support
|
||||
|
|
|
@ -47,11 +47,7 @@ pipeline {
|
|||
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
|
||||
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
|
||||
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
|
||||
// Flaky urls for different branches. Replace '-' and '.' in branch name by '_' because those
|
||||
// characters are not allowed in bash variable name.
|
||||
// Not excluding flakies from the nightly build for now.
|
||||
// EXCLUDE_TESTS_URL_master = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
|
||||
// EXCLUDE_TESTS_URL_branch_2 = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests-branch2.0/lastSuccessfulBuild/artifact/excludes/'
|
||||
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes"
|
||||
}
|
||||
parameters {
|
||||
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
pipeline {
|
||||
agent {
|
||||
node {
|
||||
label 'Hadoop'
|
||||
}
|
||||
}
|
||||
triggers {
|
||||
cron('@daily')
|
||||
}
|
||||
options {
|
||||
buildDiscarder(logRotator(numToKeepStr: '100'))
|
||||
timeout (time: 15, unit: 'MINUTES')
|
||||
timestamps()
|
||||
}
|
||||
parameters {
|
||||
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
|
||||
}
|
||||
stages {
|
||||
stage ('build flaky report') {
|
||||
steps {
|
||||
sh '''#!/usr/bin/env bash
|
||||
set -e
|
||||
if [ "${DEBUG}" = "true" ]; then
|
||||
set -x
|
||||
fi
|
||||
declare -a flaky_args
|
||||
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 5)
|
||||
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 40)
|
||||
docker build -t hbase-dev-support dev-support
|
||||
docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
// Has to be relative to WORKSPACE.
|
||||
archive "includes,excludes,dashboard.html"
|
||||
publishHTML target: [
|
||||
allowMissing: true,
|
||||
keepAll: true,
|
||||
alwaysLinkToLastBuild: true,
|
||||
// Has to be relative to WORKSPACE
|
||||
reportDir: ".",
|
||||
reportFiles: 'dashboard.html',
|
||||
reportName: 'Flaky Test Report'
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,71 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
pipeline {
|
||||
agent {
|
||||
node {
|
||||
label 'Hadoop'
|
||||
}
|
||||
}
|
||||
triggers {
|
||||
cron('@hourly')
|
||||
}
|
||||
options {
|
||||
// this should roughly match how long we tell the flaky dashboard to look at
|
||||
buildDiscarder(logRotator(numToKeepStr: '80'))
|
||||
timeout (time: 2, unit: 'HOURS')
|
||||
timestamps()
|
||||
}
|
||||
parameters {
|
||||
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
|
||||
}
|
||||
tools {
|
||||
// this should match what the yetus nightly job for the branch will use
|
||||
maven 'Maven (latest)'
|
||||
jdk "JDK 1.8 (latest)"
|
||||
}
|
||||
stages {
|
||||
stage ('run flaky tests') {
|
||||
steps {
|
||||
sh '''#!/usr/bin/env bash
|
||||
set -e
|
||||
declare -a curl_args=(--fail)
|
||||
declare -a mvn_args=(--batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dmaven.repo.local="${WORKSPACE}/local-repository")
|
||||
if [ "${DEBUG}" = "true" ]; then
|
||||
curl_args=("${curl_args[@]}" -v)
|
||||
mvn_args=("${mvn_args[@]}" -X)
|
||||
set -x
|
||||
fi
|
||||
ulimit -a
|
||||
rm -rf local-repository/org/apache/hbase
|
||||
curl "${curl_args[@]}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes"
|
||||
if [ -s includes.txt ]; then
|
||||
mvn clean package "${mvn_args[@]}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dsurefire.firstPartForkCount=3 -Dsurefire.secondPartForkCount=3
|
||||
else
|
||||
echo "set of flaky tests is currently empty."
|
||||
fi
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
junit testResults: "**/surefire-reports/*.xml", allowEmptyResults: true
|
||||
// TODO compress these logs
|
||||
archive 'includes.txt,**/surefire-reports/*,**/test-data/*'
|
||||
}
|
||||
}
|
||||
}
|
|
@ -71,10 +71,6 @@ YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
|
|||
YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}")
|
||||
YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}")
|
||||
|
||||
# Currently, flaky list is calculated only for master branch.
|
||||
UNDERSCORED_BRANCH_NAME=$(echo ${BRANCH_NAME} | tr '.-' '_')
|
||||
EXCLUDE_TESTS_URL=$(eval echo "\$EXCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
|
||||
INCLUDE_TESTS_URL=$(eval echo "\$INCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
|
||||
if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
|
||||
YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
|
||||
fi
|
||||
|
|
Loading…
Reference in New Issue