# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Github workflow that runs standard/old ITs name: "Standard ITs workflow" on: workflow_call: workflow_dispatch: jobs: changes: runs-on: ubuntu-latest # Required permissions permissions: pull-requests: read # Set job outputs to values from filter step outputs: # run everything if not a PR core: ${{ steps.filter.outputs.core || github.event_name != 'pull_request'}} common-extensions: ${{ steps.filter.outputs.common-extensions }} steps: - uses: dorny/paths-filter@v3.0.0 if: github.event_name == 'pull_request' id: filter with: filters: | common-extensions: - 'extension-core/(mysql-metadata-storage|druid-basic-security|simple-client-sslcontext|druid-testing-tools|druid-lookups-cached-global|druid-histogram|druid-datasketches|druid-parquet-extensions|druid-avro-extensions|druid-protobuf-extensions|druid-orc-extensions|druid-kafka-indexing-service|druid-s3-extensions)/**' core: - '!extension*/**' integration-index-tests-middleManager: needs: changes strategy: fail-fast: false matrix: testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction, cds-task-schema-publish-disabled, cds-coordinator-metadata-query-disabled] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 17 runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} override_config_path: ./environment-configs/test-groups/prepopulated-data use_indexer: middleManager group: ${{ matrix.testing_group }} integration-index-tests-indexer: needs: changes strategy: fail-fast: false matrix: testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, append-ingestion, compaction] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 17 runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} integration-query-tests-middleManager: needs: changes strategy: fail-fast: false matrix: testing_group: [query, query-retry, query-error, security, high-availability, centralized-datasource-schema] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 17 runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data group: ${{ matrix.testing_group }} integration-query-tests-middleManager-mariaDB: needs: changes strategy: fail-fast: false matrix: jdk: [11, 17, 21] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: ${{ matrix.jdk }} runtime_jdk: ${{ matrix.jdk }} testing_groups: -Dgroups=query use_indexer: middleManager mysql_driver: org.mariadb.jdbc.Driver override_config_path: ./environment-configs/test-groups/prepopulated-data group: query integration-shuffle-deep-store-tests: needs: changes strategy: fail-fast: false matrix: indexer: [indexer, middleManager] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 17 runtime_jdk: 17 testing_groups: -Dgroups=shuffle-deep-store use_indexer: ${{ matrix.indexer }} override_config_path: ./environment-configs/test-groups/shuffle-deep-store group: shuffle deep store integration-custom-coordinator-duties-tests: needs: changes uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 17 runtime_jdk: 17 testing_groups: -Dgroups=custom-coordinator-duties use_indexer: middleManager override_config_path: ./environment-configs/test-groups/custom-coordinator-duties group: custom coordinator duties integration-k8s-leadership-tests: needs: changes name: (Compile=openjdk17, Run=openjdk17, Cluster Build On K8s) ITNestedQueryPushDownTest integration test if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} runs-on: ubuntu-22.04 env: MVN: mvn --no-snapshot-updates MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true CONFIG_FILE: k8s_run_config_file.json IT_TEST: -Dit.test=ITNestedQueryPushDownTest POD_NAME: int-test POD_NAMESPACE: default BUILD_DRUID_CLUSTER: true steps: - name: Checkout branch uses: actions/checkout@v4 - name: setup java uses: actions/setup-java@v4 with: java-version: '17' distribution: 'zulu' # the build step produces SNAPSHOT artifacts into the local maven repository, # we include github.sha in the cache key to make it specific to that build/jdk - name: Restore Maven repository id: maven-restore uses: actions/cache/restore@v4 with: path: ~/.m2/repository key: maven-${{ runner.os }}-8-${{ github.sha }} restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }} - name: Maven build if: steps.maven-restore.outputs.cache-hit != 'true' run: | ./it.sh ci - name: Run IT id: test run: | # Debug echo echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}" MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER} - name: Debug on failure if: ${{ failure() && steps.test.conclusion == 'failure' }} run: | for v in broker middlemanager router coordinator historical ; do echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0 ||:; /usr/local/bin/kubectl get events | grep druid-tiny-cluster-"$v"s-0 ||:; done integration-other-tests: strategy: fail-fast: false matrix: indexer: [middleManager, indexer] uses: ./.github/workflows/reusable-standard-its.yml with: build_jdk: 17 runtime_jdk: 17 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema,cds-task-schema-publish-disabled,cds-coordinator-metadata-query-disabled use_indexer: ${{ matrix.indexer }} group: other