Skip tests based on files changed in the PR (#14445)

Our CI system has a lot of tests. And much of this testing is really unnecessary for most of the PRs. This PR adds some checks so we can skip these expensive tests when we know they are not necessary.
2023-06-22 12:27:23 +05:30 · 2023-06-22 12:27:23 +05:30 · f8f2fe8b7b
parent 1a9aefbb0f
commit f8f2fe8b7b
9 changed files with 106 additions and 5 deletions
--- a/.github/workflows/revised-its.yml
+++ b/.github/workflows/revised-its.yml
@ -21,7 +21,30 @@ on:
  workflow_dispatch:

 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      # run everything if not a PR
+      core: ${{ steps.filter.outputs.core || github.event_name != 'pull_request'}}
+      # the common extension in revised ITs is different from the one in standard ITs
+      common-extensions: ${{ steps.filter.outputs.common-extensions }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        if: github.event_name == 'pull_request'
+        id: filter
+        with:
+          filters: |
+            common-extensions:
+              - 'extension-core/(mysql-metadata-storage|druid-it-tools|druid-lookups-cached-global|druid-histogram|druid-datasketches|druid-parquet-extensions|druid-avro-extensions|druid-protobuf-extensions|druid-orc-extensions|druid-kafka-indexing-service|druid-s3-extensions|druid-multi-stage-query|druid-catalog)/**'
+            core:
+              - '!extension*/**'
+
  it:
+    needs: changes
    strategy:
      fail-fast: false
      matrix:
@ -31,6 +54,7 @@ jobs:
        #indexer: [indexer, middleManager]
        indexer: [middleManager]
    uses: ./.github/workflows/reusable-revised-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: ${{ matrix.jdk }}
      runtime_jdk: ${{ matrix.jdk }}
@ -40,7 +64,9 @@ jobs:
      mysql_driver: com.mysql.jdbc.Driver

  s3-deep-storage-minio:
+    needs: changes
    uses: ./.github/workflows/reusable-revised-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: 11
--- a/.github/workflows/standard-its.yml
+++ b/.github/workflows/standard-its.yml
@ -21,12 +21,35 @@ on:
  workflow_dispatch:

 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      # run everything if not a PR
+      core: ${{ steps.filter.outputs.core || github.event_name != 'pull_request'}}
+      common-extensions: ${{ steps.filter.outputs.common-extensions }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        if: github.event_name == 'pull_request'
+        id: filter
+        with:
+          filters: |
+            common-extensions:
+              - 'extension-core/(mysql-metadata-storage|druid-basic-security|simple-client-sslcontext|druid-testing-tools|druid-lookups-cached-global|druid-histogram|druid-datasketches|druid-parquet-extensions|druid-avro-extensions|druid-protobuf-extensions|druid-orc-extensions|druid-kafka-indexing-service|druid-s3-extensions)/**'
+            core:
+              - '!extension*/**'
+
  integration-index-tests-middleManager:
+    needs: changes
    strategy:
      fail-fast: false
      matrix:
        testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
    uses: ./.github/workflows/reusable-standard-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: 8
@ -35,11 +58,13 @@ jobs:
      group: ${{ matrix.testing_group }}

  integration-index-tests-indexer:
+    needs: changes
    strategy:
      fail-fast: false
      matrix:
-        testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
+        testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, append-ingestion, compaction]
    uses: ./.github/workflows/reusable-standard-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: 8
@ -48,11 +73,13 @@ jobs:
      group: ${{ matrix.testing_group }}

  integration-query-tests-middleManager:
+    needs: changes
    strategy:
      fail-fast: false
      matrix:
        testing_group: [query, query-retry, query-error, security, high-availability]
    uses: ./.github/workflows/reusable-standard-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: 8
@ -62,11 +89,13 @@ jobs:
      group: ${{ matrix.testing_group }}

  integration-query-tests-middleManager-mariaDB:
+    needs: changes
    strategy:
      fail-fast: false
      matrix:
        jdk: [8, 11]
    uses: ./.github/workflows/reusable-standard-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: ${{ matrix.jdk }}
@ -77,11 +106,13 @@ jobs:
      group: query

  integration-shuffle-deep-store-tests:
+    needs: changes
    strategy:
      fail-fast: false
      matrix:
        indexer: [indexer, middleManager]
    uses: ./.github/workflows/reusable-standard-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: 8
@ -91,7 +122,9 @@ jobs:
      group: shuffle deep store

  integration-custom-coordinator-duties-tests:
+    needs: changes
    uses: ./.github/workflows/reusable-standard-its.yml
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    with:
      build_jdk: 8
      runtime_jdk: 8
@ -101,7 +134,9 @@ jobs:
      group: custom coordinator duties

  integration-k8s-leadership-tests:
+    needs: changes
    name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
    runs-on: ubuntu-22.04
    env:
      MVN: mvn --no-snapshot-updates
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@ -31,14 +31,44 @@ on:
        description: 'Flag to decide if next tests need to run incase coverage issue failures'
        value: |
          ${{
-            (jobs.indexing_modules_test.result == 'success' || fromJson(jobs.indexing_modules_test.outputs.coverage_failure)) &&
-            (jobs.processing_modules_test.result == 'success' || fromJson(jobs.processing_modules_test.outputs.coverage_failure)) &&
-            (jobs.server_modules_test.result == 'success' || fromJson(jobs.server_modules_test.outputs.coverage_failure)) &&
-            (jobs.other_modules_test.result == 'success' || fromJson(jobs.other_modules_test.outputs.coverage_failure))
+            (jobs.indexing_modules_test.result == 'success' || jobs.indexing_modules_test.result == 'skipped' || 
+          fromJson(jobs.indexing_modules_test.outputs.coverage_failure)) &&
+            (jobs.processing_modules_test.result == 'success' || jobs.processing_modules_test.result == 'skipped' || 
+          fromJson(jobs.processing_modules_test.outputs.coverage_failure)) &&
+            (jobs.server_modules_test.result == 'success' || jobs.server_modules_test.result == 'skipped' || fromJson
+          (jobs.server_modules_test.outputs.coverage_failure)) &&
+            (jobs.other_modules_test.result == 'success' || jobs.other_modules_test.result == 'skipped' || fromJson(jobs
+          .other_modules_test.outputs.coverage_failure))
          }}

 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      kafka: ${{ steps.filter.outputs.kafka }}
+      kinesis: ${{ steps.filter.outputs.kinesis }}
+      # run everything if not a PR
+      core: ${{ steps.filter.outputs.core  || github.event_name != 'pull_request'}}
+    steps:
+      - uses: dorny/paths-filter@v2
+        if: github.event_name == 'pull_request'
+        id: filter
+        with:
+          filters: |
+            core:
+              - '!extension*/**'
+            kafka:
+              - 'extensions-core/kafka-indexing-service/**'
+            kinesis:
+              - 'extensions-core/kinesis-indexing-service/**'
+
  indexing_modules_test:
+    needs: changes
+    if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.kafka == 'true' || needs.changes.outputs.kinesis == 'true'}}
    uses: ./.github/workflows/reusable-unit-tests.yml
    with:
      jdk: ${{ inputs.jdk }}
@ -47,6 +77,8 @@ jobs:
      maven_projects: 'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'

  processing_modules_test:
+    needs: changes
+    if: ${{ needs.changes.outputs.core == 'true' }}
    uses: ./.github/workflows/reusable-unit-tests.yml
    with:
      jdk: ${{ inputs.jdk }}
@ -55,6 +87,8 @@ jobs:
      maven_projects: 'processing'

  server_modules_test:
+    needs: changes
+    if: ${{ needs.changes.outputs.core == 'true' }}
    uses: ./.github/workflows/reusable-unit-tests.yml
    with:
      jdk: ${{ inputs.jdk }}
--- a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env
+++ b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env
@ -49,6 +49,7 @@ DRUID_INSTANCE=
 # variables: druid_standard_loadList defined here, and druid_test_loadList, defined
 # in a docker-compose.yaml file, for any test-specific extensions.
 # See compose.md for more details.
+# If you are making a change in load list below, make the necessary changes in github actions too
 druid_standard_loadList=mysql-metadata-storage,druid-it-tools,druid-lookups-cached-global,druid-histogram,druid-datasketches,druid-parquet-extensions,druid-avro-extensions,druid-protobuf-extensions,druid-orc-extensions,druid-kafka-indexing-service,druid-s3-extensions,druid-multi-stage-query,druid-catalog

 # Location of Hadoop dependencies provided at runtime in the shared directory.
--- a/integration-tests/docker/environment-configs/common
+++ b/integration-tests/docker/environment-configs/common
@ -26,6 +26,7 @@ COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configu
 DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar

 # Druid configs
+# If you are making a change in load list below, make the necessary changes in github actions too
 druid_extensions_loadList=["mysql-metadata-storage","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions"]
 druid_startup_logging_logProperties=true
 druid_extensions_directory=/shared/docker/extensions
--- a/integration-tests/docker/environment-configs/common-ldap
+++ b/integration-tests/docker/environment-configs/common-ldap
@ -27,6 +27,7 @@ COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configu
 DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar

 # Druid configs
+# If you are making a change in load list below, make the necessary changes in github actions too
 druid_extensions_loadList=["mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"]
 druid_extensions_directory=/shared/docker/extensions
 druid_auth_authenticator_ldap_authorizerName=ldapauth
--- a/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties
+++ b/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties
@ -17,6 +17,7 @@
 # under the License.
 #

+# If you are making a change in load list below, make the necessary changes in github actions too
 druid_extensions_loadList=["druid-kafka-indexing-service","mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"]

 druid_coordinator_period_metadataStoreManagementPeriod=PT1H
--- a/integration-tests/docker/environment-configs/test-groups/prepopulated-data
+++ b/integration-tests/docker/environment-configs/test-groups/prepopulated-data
@ -19,6 +19,7 @@

 AWS_REGION=us-east-1

+# If you are making a change in load list below, make the necessary changes in github actions too
 druid_extensions_loadList=["mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-integration-tests"]

 # Setting s3 credentials and region to use pre-populated data for testing.
--- a/integration-tests/docker/environment-configs/test-groups/shuffle-deep-store
+++ b/integration-tests/docker/environment-configs/test-groups/shuffle-deep-store
@ -19,5 +19,6 @@

 # Test with deep storage as intermediate location to store shuffle data
 # Local deep storage will be used here
+# If you are making a change in load list below, make the necessary changes in github actions too
 druid_extensions_loadList=["mysql-metadata-storage","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"]
 druid_processing_intermediaryData_storage_type=deepstore