HBASE-19189 Ad-hoc test job for running a subset of tests lots of times
Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
Sean Busbey
parent
5bb2a24984
commit
b71544fe4d
|
|
@ -0,0 +1,93 @@
|
||||||
|
// Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
// or more contributor license agreements. See the NOTICE file
|
||||||
|
// distributed with this work for additional information
|
||||||
|
// regarding copyright ownership. The ASF licenses this file
|
||||||
|
// to you under the Apache License, Version 2.0 (the
|
||||||
|
// "License"); you may not use this file except in compliance
|
||||||
|
// with the License. You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing,
|
||||||
|
// software distributed under the License is distributed on an
|
||||||
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
// KIND, either express or implied. See the License for the
|
||||||
|
// specific language governing permissions and limitations
|
||||||
|
// under the License.
|
||||||
|
pipeline {
|
||||||
|
parameters {
|
||||||
|
string(name: 'tests', description: 'space separated list of tests to run. e.g. ' +
|
||||||
|
'TestLogRollingNoCluster TestMetricRegistryImpl TestConstraints')
|
||||||
|
string(name: 'node', defaultValue: 'Hadoop',
|
||||||
|
description: 'the node label that should be used to run the test.')
|
||||||
|
string(name: 'repeat_count', defaultValue: '100',
|
||||||
|
description: 'number of iterations to run looking for a failure.')
|
||||||
|
string(name: 'fork_count', defaultValue: '0.5C', description: '''
|
||||||
|
Given to surefire to set the number of parallel forks for a given test attempt (i.e. one
|
||||||
|
maven invocation that has all of the specified tests. The default tries to use half of the
|
||||||
|
available cores on the system.
|
||||||
|
|
||||||
|
For more information see
|
||||||
|
<a href="http://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#forkCount">
|
||||||
|
the surefire docs on the forkCount parameter</a>
|
||||||
|
''')
|
||||||
|
}
|
||||||
|
agent {
|
||||||
|
node {
|
||||||
|
label "${params.node}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
options {
|
||||||
|
timeout (time: 6, unit: 'HOURS')
|
||||||
|
timestamps()
|
||||||
|
}
|
||||||
|
environment {
|
||||||
|
// where we check out to across stages
|
||||||
|
BASEDIR = "${env.WORKSPACE}/component"
|
||||||
|
OUTPUT_RELATIVE = 'output'
|
||||||
|
OUTPUTDIR = "${env.WORKSPACE}/output"
|
||||||
|
BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
|
||||||
|
}
|
||||||
|
stages {
|
||||||
|
stage ('run tests') {
|
||||||
|
tools {
|
||||||
|
maven 'Maven (latest)'
|
||||||
|
// this needs to be set to the jdk that ought to be used to build releases on the branch
|
||||||
|
// the Jenkinsfile is stored in.
|
||||||
|
jdk "JDK 1.8 (latest)"
|
||||||
|
}
|
||||||
|
steps {
|
||||||
|
sh """#!/bin/bash -e
|
||||||
|
echo "Setting up directories"
|
||||||
|
rm -rf "${env.OUTPUTDIR}" && mkdir "${env.OUTPUTDIR}"
|
||||||
|
rm -rf ".m2-repo" && mkdir ".m2-repo"
|
||||||
|
mkdir "${env.OUTPUTDIR}/machine"
|
||||||
|
"""
|
||||||
|
sh """#!/bin/bash -e
|
||||||
|
"${env.BASEDIR}/dev-support/gather_machine_environment.sh" \
|
||||||
|
"${OUTPUT_RELATIVE}/machine"
|
||||||
|
"""
|
||||||
|
dir ("component") {
|
||||||
|
sh '''#!/bin/bash -e
|
||||||
|
./dev-support/adhoc_run_tests/adhoc_run_tests.sh \
|
||||||
|
--force-timeout 1800 \
|
||||||
|
--maven-local-repo ".m2-repo" \
|
||||||
|
--log-output "${OUTPUTDIR}" \
|
||||||
|
--surefire-fork-count "${fork_count}" \
|
||||||
|
--repeat "${repeat_count}" \
|
||||||
|
"${tests}"
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
post {
|
||||||
|
always {
|
||||||
|
archive 'output/*'
|
||||||
|
archive 'output/**/*'
|
||||||
|
}
|
||||||
|
failure {
|
||||||
|
archive 'component/**/target/surefire-reports/*'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,102 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
function usage {
|
||||||
|
echo "Usage: ${0} [options] TestSomeTestName [TestOtherTest...]"
|
||||||
|
echo ""
|
||||||
|
echo " --repeat times number of times to repeat if successful"
|
||||||
|
echo " --force-timeout seconds Seconds to wait before killing a given test run."
|
||||||
|
echo " --maven-local-repo /path/to/use Path for maven artifacts while building"
|
||||||
|
echo " --surefire-fork-count set the fork-count. only useful if multiple " \
|
||||||
|
"tests running (default 0.5C)"
|
||||||
|
echo " --log-output /path/to/use path to directory to hold attempt log"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
# Get arguments
|
||||||
|
declare -i force_timeout=7200
|
||||||
|
declare fork_count="0.5C"
|
||||||
|
declare -i attempts=1
|
||||||
|
declare maven_repo="${HOME}/.m2/repository"
|
||||||
|
declare output="."
|
||||||
|
while [ $# -gt 0 ]
|
||||||
|
do
|
||||||
|
case "$1" in
|
||||||
|
--force-timeout) shift; force_timeout=$1; shift;;
|
||||||
|
--maven-local-repo) shift; maven_repo=$1; shift;;
|
||||||
|
--repeat) shift; attempts=$1; shift;;
|
||||||
|
--log-output) shift; output=$1; shift;;
|
||||||
|
--surefire-fork-count) shift; fork_count=$1; shift;;
|
||||||
|
--) shift; break;;
|
||||||
|
-*) usage ;;
|
||||||
|
*) break;; # terminate while loop
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$#" -lt 1 ]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
function find_modules
|
||||||
|
{
|
||||||
|
declare testmaybepattern=$1
|
||||||
|
declare path
|
||||||
|
while IFS= read -r -d $'\0' path; do
|
||||||
|
while [ -n "${path}" ]; do
|
||||||
|
path=$(dirname "${path}")
|
||||||
|
if [ -f "${path}/pom.xml" ]; then
|
||||||
|
echo "${path}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done < <(find . -name "${testmaybepattern}.java" -a -type f -a -not -path '*/target/*' -print0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function echo_run_redirect
|
||||||
|
{
|
||||||
|
declare log=$1
|
||||||
|
shift
|
||||||
|
echo "${*}" >"${log}"
|
||||||
|
"${@}" >>"${log}" 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
declare -a modules
|
||||||
|
|
||||||
|
for test in "${@}"; do
|
||||||
|
for module in $(find_modules "${test}"); do
|
||||||
|
if [[ ! "${modules[*]}" =~ ${module} ]]; then
|
||||||
|
echo "adding module '${module}' to set."
|
||||||
|
modules+=(${module})
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
declare -a mvn_module_arg
|
||||||
|
|
||||||
|
for module in "${modules[@]}"; do
|
||||||
|
mvn_module_arg+=(-pl "${module}")
|
||||||
|
done
|
||||||
|
declare tests="${*}"
|
||||||
|
for attempt in $(seq "${attempts}"); do
|
||||||
|
echo "Attempt ${attempt}" >&2
|
||||||
|
echo_run_redirect "${output}/mvn_test.log" mvn --batch-mode -Dmaven.repo.local="${maven_repo}" \
|
||||||
|
-Dtest="${tests// /,}" \
|
||||||
|
-Dsurefire.rerunFailingTestsCount=0 -Dsurefire.parallel.forcedTimeout="${force_timeout}" \
|
||||||
|
-Dsurefire.shutdown=kill -DtrimStackTrace=false -am "${mvn_module_arg[@]}" \
|
||||||
|
-DforkCount="${fork_count}" package
|
||||||
|
done
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
function usage {
|
||||||
|
echo "Usage: ${0} /path/for/output/dir"
|
||||||
|
echo ""
|
||||||
|
echo " Gather info about a build machine that test harnesses should poll before running."
|
||||||
|
echo " presumes you'll then archive the passed output dir."
|
||||||
|
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$#" -lt 1 ]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
declare output=$1
|
||||||
|
|
||||||
|
echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
|
||||||
|
echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
|
||||||
|
ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
|
||||||
|
echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
|
||||||
|
mvn --offline --version >"${output}/mvn_version" 2>&1 || true
|
||||||
|
cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
|
||||||
|
cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
|
||||||
|
cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
|
||||||
|
cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
|
||||||
|
df -h >"${output}/df-h" 2>&1 || true
|
||||||
|
ps -Aww >"${output}/ps-Aww" 2>&1 || true
|
||||||
|
ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
|
||||||
|
lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
|
||||||
|
lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
|
||||||
|
ulimit -l >"${output}/ulimit-l" 2>&1 || true
|
||||||
Loading…
Reference in New Issue