From df8b16369f9d5bd890b889df02b0082960330b64 Mon Sep 17 00:00:00 2001 From: Christian Beikov Date: Tue, 22 Oct 2024 16:21:25 +0200 Subject: [PATCH] Rework/simplify GH Actions jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use `pull_request` instead of `pull_request_target` * Move Develocity build scan publishing for untrusted code to a separate workflow * Merge Atlas workflow into the main CI workflow * Split caches between trusted and untrusted code * Update secrets to use "Develocity" name instead of "Gradle Enterprise" * Update comments Co-Authored-By: Yoann Rodière --- .github/workflows/atlas.yml | 117 --------------- .github/workflows/ci-report.yml | 72 +++++++++ .github/workflows/ci.yml | 187 ++++++++++++++++++++++++ .github/workflows/contributor-build.yml | 125 ---------------- 4 files changed, 259 insertions(+), 242 deletions(-) delete mode 100644 .github/workflows/atlas.yml create mode 100644 .github/workflows/ci-report.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/contributor-build.yml diff --git a/.github/workflows/atlas.yml b/.github/workflows/atlas.yml deleted file mode 100644 index d951494470..0000000000 --- a/.github/workflows/atlas.yml +++ /dev/null @@ -1,117 +0,0 @@ -# The main CI of Hibernate ORM is https://ci.hibernate.org/job/hibernate-orm-pipeline/. -# However, Hibernate ORM builds run on GitHub actions regularly -# to check that it still works and can be used in GitHub forks. -# See https://docs.github.com/en/free-pro-team@latest/actions -# for more information about GitHub actions. - -name: Hibernate ORM build-Atlas - -on: - push: - branches: - - 'main' - # WARNING: Using pull_request_target to access secrets, but we check out the PR head commit. - # See checkout action for details. - pull_request_target: - branches: - - 'main' - -permissions: {} # none - -# See https://github.com/hibernate/hibernate-orm/pull/4615 for a description of the behavior we're getting. -concurrency: - # Consider that two builds are in the same concurrency group (cannot run concurrently) - # if they use the same workflow and are about the same branch ("ref") or pull request. - group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}" - # Cancel previous builds in the same concurrency group even if they are in process - # for pull requests or pushes to forks (not the upstream repository). - cancel-in-progress: ${{ github.event_name == 'pull_request_target' || github.repository != 'hibernate/hibernate-orm' }} - -jobs: - build: - permissions: - contents: read - name: ORM - # runs-on: ubuntu-latest - runs-on: [self-hosted, Linux, X64, OCI] - strategy: - fail-fast: false - matrix: - include: - - rdbms: oracle_atps - - rdbms: oracle_db19c - - rdbms: oracle_db21c - - rdbms: oracle_db23c - steps: - - name: Check out commit already pushed to branch - if: "! github.event.pull_request.number" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Check out PR head - uses: actions/checkout@v4 - if: github.event.pull_request.number - with: - # WARNING: This is potentially dangerous since we're checking out unreviewed code, - # and since we're using the pull_request_target event we can use secrets. - # Thus, we must be extra careful to never expose secrets to steps that execute this code, - # and to strictly limit our of secrets to those that only pose minor security threats. - # This means in particular we won't expose Develocity credentials to the main gradle executions, - # but instead will execute gradle a second time just to push build scans to Develocity; - # see below. - ref: "refs/pull/${{ github.event.pull_request.number }}/head" - persist-credentials: false - - name: Reclaim Disk Space - run: .github/ci-prerequisites.sh - - name: Start database - env: - RDBMS: ${{ matrix.rdbms }} - RUNID: ${{ github.run_number }} - run: ci/database-start.sh - - name: Set up Java 17 - uses: graalvm/setup-graalvm@v1 - with: - distribution: 'graalvm' - java-version: '21' - - name: Get year/month for cache key - id: get-date - run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - shell: bash - - name: Cache Maven local repository - uses: actions/cache@v4 - id: cache-maven - with: - path: | - ~/.m2/repository - ~/.gradle/caches/ - ~/.gradle/wrapper/ - # refresh cache every month to avoid unlimited growth - key: maven-localrepo-${{ steps.get-date.outputs.yearmonth }} - - name: Run build script - env: - RDBMS: ${{ matrix.rdbms }} - RUNID: ${{ github.run_number }} - # WARNING: exposes secrets, so must only be passed to a step that doesn't run unapproved code. - # WARNING: As this runs on untrusted nodes, we use the same access key as for PRs: - # it has limited access, essentially it can only push build scans. - DEVELOCITY_ACCESS_KEY: "${{ github.event_name == 'push' && secrets.GRADLE_ENTERPRISE_ACCESS_KEY_PR || '' }}" - run: ./ci/build-github.sh - shell: bash - - name: Publish Develocity build scan for previous build - # Don't fail a build if publishing fails - continue-on-error: true - if: "${{ !cancelled() && github.event_name == 'pull_request_target' && github.repository == 'hibernate/hibernate-orm' }}" - run: | - ./gradlew buildScanPublishPrevious - env: - # WARNING: exposes secrets, so must only be passed to a step that doesn't run unapproved code. - DEVELOCITY_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY_PR }} - - name: Upload test reports (if Gradle failed) - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test-reports-java11-${{ matrix.rdbms }} - path: | - ./**/target/reports/tests/ - - name: Omit produced artifacts from build cache - run: ./ci/before-cache.sh \ No newline at end of file diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml new file mode 100644 index 0000000000..2d28b3f6e3 --- /dev/null +++ b/.github/workflows/ci-report.yml @@ -0,0 +1,72 @@ +name: GH Actions CI reporting + +on: + workflow_run: + workflows: [ "GH Actions CI" ] + types: [ completed ] + +defaults: + run: + shell: bash + +jobs: + publish-build-scans: + name: Publish Develocity build scans + if: github.repository == 'hibernate/hibernate-orm' && github.event.workflow_run.conclusion != 'cancelled' + runs-on: ubuntu-latest + steps: + # Checkout target branch which has trusted code + - name: Check out target branch + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ github.ref }} + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + - name: Get year/month for cache key + id: get-date + run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT + shell: bash + # Note we only restore the caches, we never populate them + - name: Restore Maven/Gradle local caches + uses: actions/cache/restore@v4 + id: cache-maven-gradle + with: + path: | + ~/.m2/repository/ + ~/.m2/wrapper/ + ~/.gradle/caches/ + ~/.gradle/wrapper/ + # refresh cache every month to avoid unlimited growth + # use a different key than workflows running untrusted code + key: trusted-maven-gradle-caches-${{ steps.get-date.outputs.yearmonth }} + - name: Download GitHub Actions artifacts for the Develocity build scans + id: downloadBuildScan + uses: actions/download-artifact@v4 + with: + name: build-scan-data-${{ matrix.rdbms }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + path: /tmp/downloaded-build-scan-data/ + pattern: build-scan-data-* + # Don't fail the build if there are no matching artifacts + continue-on-error: true + - name: Publish Develocity build scans for previous builds + if: ${{ steps.downloadBuildScan.outcome != 'failure'}} + run: | + shopt -s nullglob # Don't run the loop below if there are no artifacts + status=0 + for build_scan_data_directory in /tmp/downloaded-build-scan-data/* + do + rm -rf ~/.gradle/build-scan-data + mkdir -p ~/.gradle/build-scan-data + tar -xzf "$build_scan_data_directory/build-scan-data.tgz" -C ~/.gradle/build-scan-data \ + && ./gradlew --no-build-cache buildScanPublishPrevious || status=1 + done + exit $status + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY_PR }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..e25427e5c6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,187 @@ +name: GH Actions CI + +on: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + +permissions: {} # none + +# See https://github.com/hibernate/hibernate-orm/pull/4615 for a description of the behavior we're getting. +concurrency: + # Consider that two builds are in the same concurrency group (cannot run concurrently) + # if they use the same workflow and are about the same branch ("ref") or pull request. + group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}" + # Cancel previous builds in the same concurrency group even if they are in progress + # for pull requests or pushes to forks (not the upstream repository). + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.repository != 'hibernate/hibernate-orm' }} + +jobs: + + # Main job for h2/docker DBs. + build: + permissions: + contents: read + name: OpenJDK 17 - ${{matrix.rdbms}} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - rdbms: h2 + - rdbms: hsqldb + - rdbms: mysql + - rdbms: mariadb + - rdbms: postgresql + - rdbms: edb + - rdbms: oracle + - rdbms: db2 + - rdbms: mssql + - rdbms: sybase +# Running with CockroachDB requires at least 2-4 vCPUs, which we don't have on GH Actions runners +# - rdbms: cockroachdb +# Running with HANA requires at least 8GB memory just for the database, which we don't have on GH Actions runners +# - rdbms: hana + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Reclaim Disk Space + run: .github/ci-prerequisites.sh + - name: Start database + env: + RDBMS: ${{ matrix.rdbms }} + run: ci/database-start.sh + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + - name: Get year/month for cache key + id: get-date + run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT + shell: bash + - name: Cache Maven/Gradle local caches + uses: actions/cache@v4 + id: cache-maven-gradle + with: + path: | + ~/.m2/repository/ + ~/.m2/wrapper/ + ~/.gradle/caches/ + ~/.gradle/wrapper/ + # refresh cache every month to avoid unlimited growth + # use a different key depending on whether we run in trusted or untrusted mode + key: ${{ github.event_name == 'push' && 'trusted' || 'untrusted' }}-maven-gradle-caches-${{ steps.get-date.outputs.yearmonth }} + - name: Run build script + run: ./ci/build-github.sh + shell: bash + env: + RDBMS: ${{ matrix.rdbms }} + # For jobs running on 'push', publish build scan and cache immediately. + # This won't work for pull requests, since they don't have access to secrets. + POPULATE_REMOTE_GRADLE_CACHE: ${{ github.event_name == 'push' && github.repository == 'hibernate/hibernate-orm' && 'true' || 'false' }} + DEVELOCITY_ACCESS_KEY: "${{ secrets.DEVELOCITY_ACCESS_KEY }}" + + # For jobs running on 'pull_request', tar and upload build scan data. + # The actual publishing must be done in a separate job (see ci-report.yml). + # We don't write to the remote cache as that would be unsafe. + - name: Tar build scan content pushed to subsequent jobs + if: "${{ github.event_name == 'pull_request' && !cancelled() }}" + run: tar -czf build-scan-data.tgz -C ~/.gradle/build-scan-data . + - name: Upload GitHub Actions artifact for the Develocity build scan + uses: actions/upload-artifact@v4 + if: "${{ github.event_name == 'pull_request' && !cancelled() }}" + with: + name: build-scan-data-${{ matrix.rdbms }} + path: build-scan-data.tgz + + - name: Upload test reports (if Gradle failed) + uses: actions/upload-artifact@v4 + if: failure() + with: + name: test-reports-java11-${{ matrix.rdbms }} + path: | + ./**/target/reports/tests/ + - name: Omit produced artifacts from build cache + run: ./ci/before-cache.sh + + # Job for builds on Atlas (Oracle) infrastructure. + # This is untrusted, even for pushes, see below. + atlas: + permissions: + contents: read + name: GraalVM 21 - ${{matrix.rdbms}} + # runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64, OCI] + strategy: + fail-fast: false + matrix: + include: + - rdbms: oracle_atps + - rdbms: oracle_db19c + - rdbms: oracle_db21c + - rdbms: oracle_db23c + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Reclaim Disk Space + run: .github/ci-prerequisites.sh + - name: Start database + env: + RDBMS: ${{ matrix.rdbms }} + RUNID: ${{ github.run_number }} + run: ci/database-start.sh + - name: Set up Java 21 + uses: graalvm/setup-graalvm@v1 + with: + distribution: 'graalvm' + java-version: '21' + - name: Get year/month for cache key + id: get-date + run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT + shell: bash + - name: Cache Maven/Gradle local caches + uses: actions/cache@v4 + id: cache-maven-gradle + with: + path: | + ~/.m2/repository/ + ~/.m2/wrapper/ + ~/.gradle/caches/ + ~/.gradle/wrapper/ + # refresh cache every month to avoid unlimited growth + # use a different key than jobs running in trusted mode + key: untrusted-maven-gradle-caches-${{ steps.get-date.outputs.yearmonth }} + - name: Run build script + env: + RDBMS: ${{ matrix.rdbms }} + RUNID: ${{ github.run_number }} + run: ./ci/build-github.sh + shell: bash + # Tar and upload build scan data. + # The actual publishing must be done in a separate job (see ci-report.yml). + # We don't write to the remote cache as that would be unsafe. + # That's even on push, because we do not trust Atlas runners to hold secrets: they are shared infrastructure. + - name: Tar build scan content pushed to subsequent jobs + if: "${{ !cancelled() }}" + run: tar -czf build-scan-data.tgz -C ~/.gradle/build-scan-data . + - name: Upload GitHub Actions artifact for the Develocity build scan + uses: actions/upload-artifact@v4 + if: "${{ !cancelled() }}" + with: + name: build-scan-data-${{ matrix.rdbms }} + path: build-scan-data.tgz + - name: Upload test reports (if Gradle failed) + uses: actions/upload-artifact@v4 + if: failure() + with: + name: test-reports-java11-${{ matrix.rdbms }} + path: | + ./**/target/reports/tests/ + - name: Omit produced artifacts from build cache + run: ./ci/before-cache.sh \ No newline at end of file diff --git a/.github/workflows/contributor-build.yml b/.github/workflows/contributor-build.yml deleted file mode 100644 index b293b047d0..0000000000 --- a/.github/workflows/contributor-build.yml +++ /dev/null @@ -1,125 +0,0 @@ -# The main CI of Hibernate ORM is https://ci.hibernate.org/job/hibernate-orm-pipeline/. -# However, Hibernate ORM builds run on GitHub actions regularly -# to check that it still works and can be used in GitHub forks. -# See https://docs.github.com/en/free-pro-team@latest/actions -# for more information about GitHub actions. - -name: Hibernate ORM build - -on: - push: - branches: - - 'main' - # WARNING: Using pull_request_target to access secrets, but we check out the PR head commit. - # See checkout action for details. - pull_request_target: - branches: - - 'main' - - -permissions: {} # none - -# See https://github.com/hibernate/hibernate-orm/pull/4615 for a description of the behavior we're getting. -concurrency: - # Consider that two builds are in the same concurrency group (cannot run concurrently) - # if they use the same workflow and are about the same branch ("ref") or pull request. - group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}" - # Cancel previous builds in the same concurrency group even if they are in process - # for pull requests or pushes to forks (not the upstream repository). - cancel-in-progress: ${{ github.event_name == 'pull_request_target' || github.repository != 'hibernate/hibernate-orm' }} - -jobs: - build: - permissions: - contents: read - name: Java 17 - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - rdbms: h2 - - rdbms: hsqldb - - rdbms: mysql - - rdbms: mariadb - - rdbms: postgresql - - rdbms: edb - - rdbms: oracle - - rdbms: db2 - - rdbms: mssql - - rdbms: sybase -# Running with CockroachDB requires at least 2-4 vCPUs, which we don't have on GH Actions runners -# - rdbms: cockroachdb -# Running with HANA requires at least 8GB memory just for the database, which we don't have on GH Actions runners -# - rdbms: hana - steps: - - name: Check out commit already pushed to branch - if: "! github.event.pull_request.number" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Check out PR head - uses: actions/checkout@v4 - if: github.event.pull_request.number - with: - # WARNING: This is potentially dangerous since we're checking out unreviewed code, - # and since we're using the pull_request_target event we can use secrets. - # Thus, we must be extra careful to never expose secrets to steps that execute this code, - # and to strictly limit our of secrets to those that only pose minor security threats. - # This means in particular we won't expose Develocity credentials to the main gradle executions, - # but instead will execute gradle a second time just to push build scans to Develocity; - # see below. - ref: "refs/pull/${{ github.event.pull_request.number }}/head" - persist-credentials: false - - name: Reclaim Disk Space - run: .github/ci-prerequisites.sh - - name: Start database - env: - RDBMS: ${{ matrix.rdbms }} - run: ci/database-start.sh - - name: Set up Java 17 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - - name: Get year/month for cache key - id: get-date - run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - shell: bash - - name: Cache Maven local repository - uses: actions/cache@v4 - id: cache-maven - with: - path: | - ~/.m2/repository - ~/.gradle/caches/ - ~/.gradle/wrapper/ - # refresh cache every month to avoid unlimited growth - key: maven-localrepo-${{ steps.get-date.outputs.yearmonth }} - - name: Run build script - env: - RDBMS: ${{ matrix.rdbms }} - # Don't populate Develocity cache in pull requests as that's potentially dangerous - POPULATE_REMOTE_GRADLE_CACHE: "${{ github.event_name == 'push' }}" - # WARNING: exposes secrets, so must only be passed to a step that doesn't run unapproved code. - DEVELOCITY_ACCESS_KEY: "${{ github.event_name == 'push' && secrets.GRADLE_ENTERPRISE_ACCESS_KEY || '' }}" - run: ./ci/build-github.sh - shell: bash - - name: Publish Develocity build scan for previous build (pull request) - # Don't fail a build if publishing fails - continue-on-error: true - if: "${{ !cancelled() && github.event_name == 'pull_request_target' && github.repository == 'hibernate/hibernate-orm' }}" - run: | - ./gradlew buildScanPublishPrevious - env: - # WARNING: exposes secrets, so must only be passed to a step that doesn't run unapproved code. - DEVELOCITY_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY_PR }} - - name: Upload test reports (if Gradle failed) - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test-reports-java11-${{ matrix.rdbms }} - path: | - ./**/target/reports/tests/ - - name: Omit produced artifacts from build cache - run: ./ci/before-cache.sh