HADOOP-17807. Use separate src dir for platform builds (#3210)

This commit is contained in:
Gautham B A 2021-07-27 01:39:36 +05:30 committed by GitHub
parent d710ec8d85
commit 97c88c97de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 196 additions and 71 deletions

View File

@ -29,9 +29,6 @@ pipeline {
}
environment {
SOURCEDIR = 'src'
// will also need to change notification section below
PATCHDIR = 'out'
YETUS='yetus'
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
@ -56,6 +53,35 @@ pipeline {
}
}
// Setup codebase so that each platform's build happens in its own exclusive copy of the
// codebase.
// Primarily because YETUS messes up the git branch information and affects the subsequent
// optional stages after the first one.
stage ('setup sources') {
steps {
dir("${WORKSPACE}/centos-8") {
sh '''#!/usr/bin/env bash
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
'''
}
dir("${WORKSPACE}/debian-10") {
sh '''#!/usr/bin/env bash
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
'''
}
dir("${WORKSPACE}/ubuntu-focal") {
sh '''#!/usr/bin/env bash
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
'''
}
}
}
// This is an optional stage which runs only when there's a change in
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
@ -63,6 +89,8 @@ pipeline {
// break the Hadoop build on Centos 8.
stage ('precommit-run Centos 8') {
environment {
SOURCEDIR = "${WORKSPACE}/centos-8/src"
PATCHDIR = "${WORKSPACE}/centos-8/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
IS_OPTIONAL = 1
}
@ -78,10 +106,32 @@ pipeline {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
'''
}
}
post {
// Since this is an optional platform, we want to copy the artifacts
// and archive it only if the build fails, to help with debugging.
failure {
sh '''#!/usr/bin/env bash
cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}
cleanup() {
script {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
'''
}
}
}
}
// This is an optional stage which runs only when there's a change in
@ -91,6 +141,8 @@ pipeline {
// break the Hadoop build on Debian 10.
stage ('precommit-run Debian 10') {
environment {
SOURCEDIR = "${WORKSPACE}/debian-10/src"
PATCHDIR = "${WORKSPACE}/debian-10/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
IS_OPTIONAL = 1
}
@ -106,16 +158,40 @@ pipeline {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
'''
}
}
post {
// Since this is an optional platform, we want to copy the artifacts
// and archive it only if the build fails, to help with debugging.
failure {
sh '''#!/usr/bin/env bash
cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}
cleanup() {
script {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
'''
}
}
}
}
// We want to use Ubuntu Focal as our main CI and thus, this stage
// isn't optional (runs for all the PRs).
stage ('precommit-run Ubuntu focal') {
environment {
SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
IS_OPTIONAL = 0
}
@ -131,69 +207,76 @@ pipeline {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
'''
}
}
}
post {
always {
script {
// Publish status if it was missed (YETUS-1059)
withCredentials(
[usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER')]) {
sh '''#!/usr/bin/env bash
# Copy the artifacts of Ubuntu focal build to workspace
cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}"
# Send Github status
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
'''
}
// YETUS output
archiveArtifacts "out/**"
// Publish the HTML report so that it can be looked at
// Has to be relative to WORKSPACE.
publishHTML (target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: "out",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
// Publish JUnit results
try {
junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
} catch(e) {
echo 'junit processing: ' + e.toString()
}
}
}
cleanup() {
script {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
'''
}
}
}
}
}
post {
always {
script {
// Publish status if it was missed (YETUS-1059)
withCredentials(
[usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER')]) {
sh '''#!/usr/bin/env bash
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
'''
}
// Yetus output
archiveArtifacts "${env.PATCHDIR}/**"
// Publish the HTML report so that it can be looked at
// Has to be relative to WORKSPACE.
publishHTML (target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: "${env.PATCHDIR}",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
// Publish JUnit results
try {
junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
} catch(e) {
echo 'junit processing: ' + e.toString()
}
}
}
// Jenkins pipeline jobs fill slaves on PRs without this :(
cleanup() {
script {
sh '''
# See YETUS-764
if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
echo "test-patch process appears to still be running: killing"
kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
sleep 10
fi
if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
echo "test-patch container appears to still be running: killing"
docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
fi
# See HADOOP-13951
chmod -R u+rxw "${WORKSPACE}"
'''
sh '''#!/usr/bin/env bash
# See HADOOP-13951
chmod -R u+rxw "${WORKSPACE}"
'''
deleteDir()
}
}

View File

@ -115,10 +115,10 @@ function run_ci() {
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
# this must be clean for every run
if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
rm -rf "${WORKSPACE:?}/${PATCHDIR}"
if [[ -d "${PATCHDIR}" ]]; then
rm -rf "${PATCHDIR:?}"
fi
mkdir -p "${WORKSPACE}/${PATCHDIR}"
mkdir -p "${PATCHDIR}"
# if given a JIRA issue, process it. If CHANGE_URL is set
# (e.g., Github Branch Source plugin), process it.
@ -128,23 +128,23 @@ function run_ci() {
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
elif [[ -z "${CHANGE_URL}" ]]; then
echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html"
echo "Full build skipped" >"${PATCHDIR}/report.html"
exit 0
fi
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
# where the source is located
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
YETUS_ARGS+=("--basedir=${SOURCEDIR}")
# our project defaults come from a personality file
YETUS_ARGS+=("--project=hadoop")
YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh")
# lots of different output formats
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
# enable writing back to Github
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
@ -206,7 +206,49 @@ function run_ci() {
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
}
# Check if the CI needs to be run, if so, do so :)
if check_ci_run; then
run_ci
## @description Cleans up the processes started by YETUS
function cleanup_ci_proc() {
# See YETUS-764
if [ -f "${PATCHDIR}/pidfile.txt" ]; then
echo "test-patch process appears to still be running: killing"
kill "$(cat "${PATCHDIR}/pidfile.txt")" || true
sleep 10
fi
if [ -f "${PATCHDIR}/cidfile.txt" ]; then
echo "test-patch container appears to still be running: killing"
docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true
fi
}
## @description Invokes github_status_recovery in YETUS's precommit
function github_status_recovery() {
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true
}
if [ -z "$1" ]; then
echo "Must specify an argument for jenkins.sh"
echo "run_ci - Runs the CI based on platform image as defined by DOCKERFILE"
echo "cleanup_ci_proc - Cleans up the processes spawned for running the CI"
echo "github_status_recovery - Sends Github status (refer to YETUS precommit for more details)"
exit 1
fi
# Process arguments to jenkins.sh
if [ "$1" == "run_ci" ]; then
# Check if the CI needs to be run, if so, do so :)
if check_ci_run; then
run_ci
else
echo "No C++ file/C++ build/platform changes found, will not run CI"
fi
elif [ "$1" == "cleanup_ci_proc" ]; then
cleanup_ci_proc
elif [ "$1" == "github_status_recovery" ]; then
github_status_recovery
else
echo "Don't know how to process $1"
exit 1
fi