Migrate ITs from Travis to GHA (#13681)

.github/workflows/reusable-revised-its.yml

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Revised Integration Tests shared workflow
+on:
+  workflow_call:
+    inputs:
+      script:
+        description: 'Which IT script to run'
+        required: true
+        type: string
+      build_jdk:
+        description: 'Which jdk version was used to build'
+        required: true
+        type: string
+      runtime_jdk:
+        description: 'Which JDK version to use at runtime'
+        required: true
+        type: string
+      use_indexer:
+        description: 'Which indexer to use'
+        required: true
+        type: string
+      it:
+        description: 'IT test Category'
+        required: true
+        type: string
+
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test: # Github job that runs a given revised/new IT against retrieved cached druid docker image
+    name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }})
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v3
+
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ inputs.build_jdk }}
+          cache: maven
+
+      - name: Retrieve cached docker image
+        uses: actions/cache/restore@v3
+        with:
+          key: druid-container-jdk${{ inputs.build_jdk }}.tar.gz
+          path: |
+            ./druid-container-jdk${{ inputs.build_jdk }}.tar.gz
+            ./integration-tests-ex/image/target/env.sh
+
+      - name: Load docker image
+        run: |
+          docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz
+          docker images
+
+      - name: Run IT
+        run: ${{ inputs.script }}

.github/workflows/reusable-standard-its.yml

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Revised Integration Tests shared workflow
+on:
+  workflow_call:
+    inputs:
+      runtime_jdk:
+        description: 'Which JDK version to use at runtime'
+        required: true
+        type: string
+      use_indexer:
+        description: 'Which indexer to use'
+        required: true
+        type: string
+      override_config_path:
+        description: 'Existing Druid configuration (using Docker) can be overridden or new config vars can be set e.g. when using cloud storage'
+        required: false
+        type: string
+      group:
+        required: true
+        type: string
+        description: 'Name of group of tests running (to display)'
+      testing_groups:
+        required: true
+        type: string
+      build_jdk:
+        description: 'Which jdk version was used to build'
+        required: true
+        type: string
+      mysql_driver:
+        description: 'MySQL driver to use'
+        required: false
+        type: string
+        default: com.mysql.jdbc.Driver
+
+env:
+  MVN: mvn --no-snapshot-updates
+  MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
+  MAVEN_SKIP_TESTS: -P skip-tests
+  DOCKER_IP: 127.0.0.1  # for integration tests
+  MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test: # Github job that runs a given standard/old IT for a given testing group
+    name: ${{ inputs.group }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v3
+
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ inputs.runtime_jdk }}
+
+      - name: Restore Maven repository
+        id: maven-restore
+        uses: actions/cache/restore@v3
+        with:
+         path: ~/.m2/repository
+         key: maven-${{ runner.os }}-${{ inputs.build_jdk }}-${{ hashFiles('**/pom.xml') }}
+
+      - name: Run IT
+        env:
+          MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
+          MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
+        run: |
+          # Debug echo
+          echo "Mysql driver: ${MYSQL_DRIVER_CLASSNAME}"
+          echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
+          MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}

.github/workflows/reusable-unit-tests.yml

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+name: Revised Unit Tests shared workflow
+on:
+  workflow_call:
+    inputs:
+      jdk:
+        required: true
+        type: string
+        description: 'JDK version used to test Druid'
+      sql_compatibility:
+        required: false
+        type: boolean
+        default: true
+        description: 'For SQL compatibility'
+      module:
+        required: true
+        type: string
+        description: 'Name of group of tests that are running'
+      maven_projects:
+        required: true
+        type: string
+        description: 'Name of group of tests running (to display)'
+
+env:
+  MVN: mvn -B
+  MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
+  MAVEN_SKIP_TESTS: -P skip-tests
+  MAVEN_OPTS: -Xmx3000m
+  FORCE_COLOR: 2
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  unit-tests:
+    name: ${{ inputs.module }} modules test
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout branch
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: setup jdk${{ inputs.jdk }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ inputs.jdk }}
+
+      - name: Restore Maven repository
+        id: maven-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          key: maven-${{ runner.os }}-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml') }}
+
+      - name: setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.17.0
+
+      - name: setup variables
+        run: |
+          export base_ref=${{ github.base_ref }}
+          echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
+          export druid_use_default_value_for_null=${{ inputs.sql_compatibility }}
+          echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
+
+      - name: fetch base branch for test coverage
+        if: ${{ github.base_ref != '' }}
+        run: |
+          # Add merge target branch to determine diff.
+          # This is not needed for build triggered by tags, since there will be no code diff.
+          git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
+
+      - name: setup diff-test-coverage
+        run: npm install @connectis/diff-test-coverage@1.5.3
+
+      - name: test & coverage
+        env:
+          MAVEN_PROJECTS: ${{ inputs.maven_projects }}
+          MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
+        run: ./.github/scripts/unit_tests_script.sh

.github/workflows/revised-its.yml

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Github workflow that runs revised/new ITs
+on:
+  workflow_call
+
+jobs:
+  it:
+    strategy:
+      fail-fast: false
+      matrix:
+        #jdk: [8, 11, 17]
+        jdk: [8]
+        it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat]
+        #indexer: [indexer, middleManager]
+        indexer: [middleManager]
+    uses: apache/druid/.github/workflows/reusable-revised-its.yml@master
+    with:
+      build_jdk: ${{ matrix.jdk }}
+      runtime_jdk: ${{ matrix.jdk }}
+      use_indexer: ${{ matrix.indexer }}
+      script: ./it.sh github ${{ matrix.it }}
+      it: ${{ matrix.it }}

.github/workflows/standard-its.yml

@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Github workflow that runs standard/old ITs
+on:
+  workflow_call
+
+jobs:
+  integration-index-tests-middleManager:
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [8, 11]
+        testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: ${{ matrix.jdk }}
+      testing_groups: -Dgroups=${{ matrix.testing_group }}
+      use_indexer: middleManager
+      group: ${{ matrix.testing_group }}
+
+  integration-index-tests-indexer:
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [8, 11]
+        testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: ${{ matrix.jdk }}
+      testing_groups: -Dgroups=${{ matrix.testing_group }}
+      use_indexer: indexer
+      group: ${{ matrix.testing_group }}
+
+  integration-query-tests-middleManager:
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [8, 11]
+        testing_group: [query, query-retry, query-error, security, high-availability]
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: ${{ matrix.jdk }}
+      testing_groups: -Dgroups=${{ matrix.testing_group }}
+      use_indexer: middleManager
+      override_config_path: ./environment-configs/test-groups/prepopulated-data
+      group: ${{ matrix.testing_group }}
+
+  integration-query-tests-middleManager-mariaDB:
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [8, 11]
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: ${{ matrix.jdk }}
+      testing_groups: -Dgroups=query
+      use_indexer: middleManager
+      mysql_driver: org.mariadb.jdbc.Driver
+      override_config_path: ./environment-configs/test-groups/prepopulated-data
+      group: query
+
+  integration-shuffle-deep-store-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        indexer: [indexer, middleManager]
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: 8
+      testing_groups: -Dgroups=shuffle-deep-store
+      use_indexer: ${{ matrix.indexer }}
+      override_config_path: ./environment-configs/test-groups/shuffle-deep-store
+      group: shuffle deep store
+
+  integration-custom-coordinator-duties-tests:
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: 8
+      testing_groups: -Dgroups=custom-coordinator-duties
+      use_indexer: middleManager
+      override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
+      group: custom coordinator duties
+
+  integration-other-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [8, 11]
+        indexer: [middleManager, indexer]
+    uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+    with:
+      build_jdk: 8
+      runtime_jdk: ${{ matrix.jdk }}
+      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
+      use_indexer: ${{ matrix.indexer }}
+      group: other

.github/workflows/static-checks.yml

@@ -37,6 +37,7 @@ env:
 jobs:
   static-checks:
     strategy:
+      fail-fast: false
       matrix:
         java: [ 'jdk8', 'jdk11', 'jdk17' ]
     runs-on: ubuntu-latest

.github/workflows/unit-and-integration-tests-unified.yml

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Unit & Integration tests CI
+on:
+  push:
+    branches:
+      - master
+      - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+  pull_request:
+    branches:
+      - master
+      - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  build:
+    name: build (jdk${{ matrix.jdk }})
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [ '8', '11', '17' ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v3
+
+      - name: Cache Maven m2 repository
+        id: maven
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-${{ runner.os }}-${{ matrix.jdk }}
+
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ matrix.jdk }}
+
+      - name: Maven build
+        id: maven_build
+        run: |
+          ./it.sh ci
+
+      - name: Cache targets
+        id: target
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./**/target
+          key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-${{ runner.os }}-${{ matrix.jdk }}-targets
+
+      - name: Container build
+        run: |
+          ./it.sh image
+          source ./integration-tests-ex/image/target/env.sh
+          docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }}
+
+      - name: Save docker container to archive
+        run: |
+          source ./integration-tests-ex/image/target/env.sh
+          echo $DRUID_IT_IMAGE_NAME
+          docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz
+
+      - name: Cache image
+        id: docker_container
+        uses: actions/cache@v3
+        with:
+          key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ hashFiles('**/pom.xml') }}
+          path: |
+            ./druid-container-jdk${{ matrix.jdk }}.tar.gz
+            ./integration-tests-ex/image/target/env.sh
+          restore-keys: |
+            druid-container-jdk${{ matrix.jdk }}.tar.gz-
+
+  unit-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        sql_compatibility: [ false, true ]
+    name: unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})
+    uses: apache/druid/.github/workflows/unit-tests.yml@master
+    needs: build
+    with:
+      jdk: 8
+      sql_compatibility: ${{ matrix.sql_compatibility }}
+
+  unit-tests-phase2:
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        sql_compatibility: [ false, true ]
+    name: unit tests (jdk${{ matrix.jdk }}, sql-compat=${{ matrix.sql_compatibility }})
+    uses: apache/druid/.github/workflows/unit-tests.yml@master
+    needs: unit-tests
+    with:
+      jdk: ${{ matrix.jdk }}
+      sql_compatibility: ${{ matrix.sql_compatibility }}
+
+  standard-its:
+    needs: unit-tests
+    uses: apache/druid/.github/workflows/standard-its.yml@master
+
+  revised-its:
+    needs: unit-tests
+    uses: apache/druid/.github/workflows/revised-its.yml@master

.github/workflows/unit-tests.yml

@@ -11,93 +11,50 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License
 
-name: (openjdk8) Unit Tests CI
 on:
-  push:
+  workflow_call:
-    branches:
+    inputs:
-      - master
+      jdk:
-      - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+        required: true
-  pull_request:
+        type: string
-    branches:
+        description: 'JDK version used to test Druid'
-      - master
+      sql_compatibility:
-      - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+        required: false
-
+        type: boolean
-concurrency:
+        default: true
-  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+        description: 'For SQL compatibility'
-  cancel-in-progress: true
-
-env:
-  MVN: mvn -B
-  MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
-  MAVEN_SKIP_TESTS: -P skip-tests
-  MAVEN_OPTS: -Xmx3000m
-  FORCE_COLOR: 2
 
 jobs:
-  unit-tests:
+  indexing_modules_test:
-    strategy:
+    uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
-      matrix:
+    with:
-        DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
+      jdk: ${{ inputs.jdk }}
-    runs-on: ubuntu-latest
+      sql_compatibility: ${{ inputs.sql_compatibility }}
-    steps:
+      module: indexing
-      - name: checkout branch
+      maven_projects: 'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
 
-      - name: setup jdk8
+  processing_modules_test:
-        uses: actions/setup-java@v3
+    uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
-        with:
+    with:
-          distribution: 'zulu'
+      jdk: ${{ inputs.jdk }}
-          java-version: 8
+      sql_compatibility: ${{ inputs.sql_compatibility }}
-          cache: 'maven'
+      module: processing
+      maven_projects: 'processing'
 
-      - name: maven install
+  server_modules_test:
-        run: |
+    uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
-          echo 'Running Maven install...' &&
+    with:
-          ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
+      jdk: ${{ inputs.jdk }}
-          ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+      sql_compatibility: ${{ inputs.sql_compatibility }}
+      module: server
+      maven_projects: 'server'
 
-      - name: setup node
+  other_modules_test:
-        uses: actions/setup-node@v3
+    uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
-        with:
+    with:
-          node-version: 16.17.0
+      jdk: ${{ inputs.jdk }}
-
+      sql_compatibility: ${{ inputs.sql_compatibility }}
-      - name: setup variables
+      module: other
-        run: |
+      maven_projects: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
-          export base_ref=${{ github.base_ref }}
-          echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
-          export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
-          echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
-
-      - name: fetch base branch for test coverage
-        if: ${{ github.base_ref != '' }}
-        run: |
-          # Add merge target branch to determine diff.
-          # This is not needed for build triggered by tags, since there will be no code diff.
-          git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
-
-      - name: Setup diff-test-coverage
-        run: npm install @connectis/diff-test-coverage@1.5.3
-
-      - name: "(openjdk8) indexing modules test"
-        env:
-          MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
-        run: ./.github/scripts/unit_tests_script.sh
-
-      - name: "(openjdk8) processing module test"
-        env:
-          MAVEN_PROJECTS: processing
-        run: ./.github/scripts/unit_tests_script.sh
-
-      - name: "(openjdk8) server module test"
-        env:
-          MAVEN_PROJECTS: server
-        run: ./.github/scripts/unit_tests_script.sh
-
-      - name: "(openjdk8) other modules test"
-        env:
-          MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
-        run: ./.github/scripts/unit_tests_script.sh

integration-tests/docker/docker-compose.query-error-test.yml

@@ -78,7 +78,7 @@ services:
       druid-it-net:
         ipv4_address: 172.172.172.14
     ports:
-      - 8084:8083
+      - 8086:8083 #8084 is used by mono service on GHA runners
       - 8284:8283
       - 5010:5007
     privileged: true

integration-tests/docker/docker-compose.query-retry-test.yml

@@ -78,7 +78,7 @@ services:
       druid-it-net:
         ipv4_address: 172.172.172.14
     ports:
-      - 8084:8083
+      - 8086:8083 #8084 is used by mono service on GHA runners
       - 8284:8283
       - 5010:5007
     privileged: true

integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java

@@ -108,6 +108,8 @@ public class ITJdbcQueryTest
     };
     // ensure that wikipedia segments are loaded completely
     dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
+    dataLoaderHelper.waitUntilDatasourceIsReady("wikipedia");
+    dataLoaderHelper.waitUntilDatasourceIsReady("twitterstream");
   }
 
   @Test

it.sh

@@ -31,6 +31,8 @@ function usage
 {
   cat <<EOF
 Usage: $0 cmd [category]
+  ci
+      build Druid and the distribution for CI pipelines
   build
       build Druid and the distribution
   dist
@@ -49,6 +51,8 @@ Usage: $0 cmd [category]
       show the last 20 lines of each container log
   travis <category>
       run one IT in Travis (build dist, image, run test, tail logs)
+  github <category>
+      run one IT in Github Workflows (run test, tail logs)
   prune
       prune Docker volumes
 
@@ -189,6 +193,9 @@ case $CMD in
   "help" )
     usage
     ;;
+  "ci" )
+    mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE -T1.0C
+    ;;
   "build" )
     mvn clean package -P dist $MAVEN_IGNORE -T1.0C
     ;;
@@ -230,6 +237,11 @@ case $CMD in
     $0 test $CATEGORY
     $0 tail $CATEGORY
     ;;
+  "github" )
+    prepare_category $1
+    $0 test $CATEGORY
+    $0 tail $CATEGORY
+    ;;
   "prune" )
     # Caution: this removes all volumes, which is generally what you
     # want when testing.