From fb685adc9486cad52166bb0d9201423b412a41a3 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 13 Nov 2019 17:12:00 +0200 Subject: [PATCH] Move periodic job to ES repo (#48570) * Move periodic job to ES repo This change kickstarts the process of moving CI job definitions to this repo. * Added a minimal readme to provide pointers to the documentation * Update .ci/README.md Co-Authored-By: Rory Hunter * Update .ci/README.md Co-Authored-By: Rory Hunter * point to main repo * PR review * Add link to JJBB --- .ci/README.md | 16 ++++ .ci/build.sh | 32 +++++++ .ci/jobs.t/defaults.yml | 92 +++++++++++++++++++ ...c+elasticsearch+{branch}+periodic-next.yml | 11 +++ .ci/make-branch-config.sh | 11 +++ .gitignore | 3 + 6 files changed, 165 insertions(+) create mode 100644 .ci/README.md create mode 100755 .ci/build.sh create mode 100644 .ci/jobs.t/defaults.yml create mode 100644 .ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml create mode 100755 .ci/make-branch-config.sh diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000000..8f63884462a --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,16 @@ +CI configuration for Elasticsearch +================================== + +Layout and Syntax +----------------- + +CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/). +Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax +simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/). +Macros are not allowed, and each job must be defined in its own file. +Merging of the default configuration is customized so unlike in standard JJB, +it recurses into YML objects. +Further (internal) documentation on the setup +[is available](https://github.com/elastic/infra/blob/master/flavortown/jjbb/README.md) +. + diff --git a/.ci/build.sh b/.ci/build.sh new file mode 100755 index 00000000000..0a04728d872 --- /dev/null +++ b/.ci/build.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# drop page cache and kernel slab objects on linux +[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches + +rm -Rfv ~/.gradle/init.d/init.gradle +mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d + +if [ -f /proc/cpuinfo ] ; then + MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` +else + if [[ "$OSTYPE" == "darwin"* ]]; then + MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` + # Looks like it's too much for our workers so reduce it further + MAX_WORKERS=$(($MAX_WORKERS/2)) + else + echo "Unsupported OS Type: $OSTYPE" + exit 1 + fi +fi + +if pwd | grep -v -q ^/dev/shm ; then + echo "Not running on a ramdisk, reducing number of workers" + MAX_WORKERS=$(($MAX_WORKERS*2/3)) +fi + +export GRADLE_OPTS="-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m" +set -e +./gradlew --parallel --scan \ + -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \ + --parallel --max-workers=$MAX_WORKERS \ + "$@" \ No newline at end of file diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml new file mode 100644 index 00000000000..d0f13ac5400 --- /dev/null +++ b/.ci/jobs.t/defaults.yml @@ -0,0 +1,92 @@ +--- + +##### GLOBAL METADATA + +- meta: + cluster: elasticsearch-ci + +##### JOB DEFAULTS + +- job: + vault: + url: https://secrets.elastic.co:8200 + role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db + node: "general-purpose" + concurrent: true + logrotate: + daysToKeep: 30 + numToKeep: 90 + artifactDaysToKeep: 7 + scm: + - git: + name: origin + # master node jenkins user ~/.ssh + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git" + branches: + - "%BRANCH%" + url: "https://github.com/elastic/elasticsearch.git" + basedir: "" + wipe-workspace: "yes" + wrappers: + - timeout: + type: absolute + timeout: "120" + fail: true + - ansicolor + - timestamps + # TODO: No support un JJBB ? + # - gradle-build-scan + properties: + - github: + url: https://github.com/elastic/elasticsearch/ + - inject: + properties-content: | + HOME=$JENKINS_HOME + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA7_HOME=$HOME/.java/java7 + JAVA8_HOME=$HOME/.java/java8 + JAVA9_HOME=$HOME/.java/java9 + JAVA10_HOME=$HOME/.java/java10 + JAVA11_HOME=$HOME/.java/java11 + JAVA12_HOME=$HOME/.java/openjdk12 + JAVA13_HOME=$HOME/.java/openjdk13 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push + .ci/build.sh check + publishers: + - email: + recipients: infra-root+build@elastic.co + # Upload additional logs + - google-cloud-storage: + credentials-id: 'elasticsearch-ci-gcs-plugin' + uploads: + - classic: + file-pattern: 'build/*.tar.bz2' + storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME' + share-publicly: false + upload-for-failed-jobs: true + show-inline: true + # Notify homer + - postbuildscript: + builders: + - role: SLAVE + build-on: + - SUCCESS + - FAILURE + - UNSTABLE + build-steps: + - shell: | + curl -sS -X POST \ + -H "Content-Type: text/plain" \ + --data "$BUILD_URL" \ + "https://homer.app.elstc.co/webhook/jenkins/build-finished" || true + + + + diff --git a/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml b/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml new file mode 100644 index 00000000000..d4c408cb588 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml @@ -0,0 +1,11 @@ +- job: + name: elastic+elasticsearch+%BRANCH%+periodic-next + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic + display-name: "elastic / elasticsearch # %BRANCH% - periodic (experimental)" + description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n" + triggers: + - timed: "H H/1 * * *" + builders: + - shell: | + #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push + .ci/build.sh -Dbwc.checkout.align=true check diff --git a/.ci/make-branch-config.sh b/.ci/make-branch-config.sh new file mode 100755 index 00000000000..ecbdfe3f3a9 --- /dev/null +++ b/.ci/make-branch-config.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -z "$BRANCH" ] ; then + echo "BRANCH is unset" + exit 1 +fi + +rm -Rf .ci/jobs +cp -r .ci/jobs.t .ci/jobs + +sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml diff --git a/.gitignore b/.gitignore index b1a78d832e9..e3f75ba3d1f 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ eclipse-build # projects using testfixtures testfixtures_shared/ + +# These are generated from .ci/jobs.t +.ci/jobs/