druid/.github/workflows/standard-its.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Github workflow that runs standard/old ITs
on:
  workflow_call:
  workflow_dispatch:

jobs:
  integration-index-tests-middleManager:
    strategy:
      fail-fast: false
      matrix:
        jdk: [8, 11]
        testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: ${{ matrix.jdk }}
      testing_groups: -Dgroups=${{ matrix.testing_group }}
      use_indexer: middleManager
      group: ${{ matrix.testing_group }}

  integration-index-tests-indexer:
    strategy:
      fail-fast: false
      matrix:
        jdk: [8, 11]
        testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: ${{ matrix.jdk }}
      testing_groups: -Dgroups=${{ matrix.testing_group }}
      use_indexer: indexer
      group: ${{ matrix.testing_group }}

  integration-query-tests-middleManager:
    strategy:
      fail-fast: false
      matrix:
        jdk: [8, 11]
        testing_group: [query, query-retry, query-error, security, high-availability]
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: ${{ matrix.jdk }}
      testing_groups: -Dgroups=${{ matrix.testing_group }}
      use_indexer: middleManager
      override_config_path: ./environment-configs/test-groups/prepopulated-data
      group: ${{ matrix.testing_group }}

  integration-query-tests-middleManager-mariaDB:
    strategy:
      fail-fast: false
      matrix:
        jdk: [8, 11]
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: ${{ matrix.jdk }}
      testing_groups: -Dgroups=query
      use_indexer: middleManager
      mysql_driver: org.mariadb.jdbc.Driver
      override_config_path: ./environment-configs/test-groups/prepopulated-data
      group: query

  integration-shuffle-deep-store-tests:
    strategy:
      fail-fast: false
      matrix:
        indexer: [indexer, middleManager]
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: 8
      testing_groups: -Dgroups=shuffle-deep-store
      use_indexer: ${{ matrix.indexer }}
      override_config_path: ./environment-configs/test-groups/shuffle-deep-store
      group: shuffle deep store

  integration-custom-coordinator-duties-tests:
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: 8
      testing_groups: -Dgroups=custom-coordinator-duties
      use_indexer: middleManager
      override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
      group: custom coordinator duties

  integration-k8s-leadership-tests:
    name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
    runs-on: ubuntu-22.04
    env:
      MVN: mvn --no-snapshot-updates
      MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
      CONFIG_FILE: k8s_run_config_file.json
      IT_TEST: -Dit.test=ITNestedQueryPushDownTest
      POD_NAME: int-test
      POD_NAMESPACE: default
      BUILD_DRUID_CLUSTER: true
    steps:
      - name: Checkout branch
        uses: actions/checkout@v3

      - name: Setup java
        run: export JAVA_HOME=$JAVA_HOME_8_X64

      - name: Restore Maven repository
        id: maven-restore
        uses: actions/cache/restore@v3
        with:
          path: ~/.m2/repository
          key: maven-${{ runner.os }}-8-${{ github.sha }}

      - name: Maven build
        if: steps.maven-restore.outputs.cache-hit != 'true'
        run: |
          ./it.sh ci          

      - name: Run IT
        id: test
        run: |
          # Debug echo
          echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}"
          MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}          

      - name: Debug on failure
        if: ${{ failure() && steps.test.conclusion == 'failure' }}
        run: |
          for v in broker middlemanager router coordinator historical ; do
          echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";
          sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0;
          done          

  integration-other-tests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [8, 11]
        indexer: [middleManager, indexer]
    uses: ./.github/workflows/reusable-standard-its.yml
    with:
      build_jdk: 8
      runtime_jdk: ${{ matrix.jdk }}
      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
      use_indexer: ${{ matrix.indexer }}
      group: other