From c95a26cae3ecafca430326ae8302fefda59b9fbd Mon Sep 17 00:00:00 2001 From: Tejaswini Bandlamudi <96047043+tejaswini-imply@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:01:29 +0530 Subject: [PATCH] Migrate ITs from Travis to GHA (#13681) --- .github/workflows/reusable-revised-its.yml | 73 ++++++++++ .github/workflows/reusable-standard-its.yml | 86 ++++++++++++ .github/workflows/reusable-unit-tests.yml | 95 +++++++++++++ .github/workflows/revised-its.yml | 36 +++++ .github/workflows/standard-its.yml | 115 ++++++++++++++++ .github/workflows/static-checks.yml | 1 + .../unit-and-integration-tests-unified.yml | 130 ++++++++++++++++++ .github/workflows/unit-tests.yml | 123 ++++++----------- .../docker-compose.query-error-test.yml | 2 +- .../docker-compose.query-retry-test.yml | 2 +- .../druid/tests/query/ITJdbcQueryTest.java | 2 + it.sh | 12 ++ 12 files changed, 592 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/reusable-revised-its.yml create mode 100644 .github/workflows/reusable-standard-its.yml create mode 100644 .github/workflows/reusable-unit-tests.yml create mode 100644 .github/workflows/revised-its.yml create mode 100644 .github/workflows/standard-its.yml create mode 100644 .github/workflows/unit-and-integration-tests-unified.yml diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml new file mode 100644 index 00000000000..53a708add17 --- /dev/null +++ b/.github/workflows/reusable-revised-its.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Revised Integration Tests shared workflow +on: + workflow_call: + inputs: + script: + description: 'Which IT script to run' + required: true + type: string + build_jdk: + description: 'Which jdk version was used to build' + required: true + type: string + runtime_jdk: + description: 'Which JDK version to use at runtime' + required: true + type: string + use_indexer: + description: 'Which indexer to use' + required: true + type: string + it: + description: 'IT test Category' + required: true + type: string + +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + +jobs: + test: # Github job that runs a given revised/new IT against retrieved cached druid docker image + name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}) + runs-on: ubuntu-22.04 + steps: + - name: Checkout branch + uses: actions/checkout@v3 + + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ inputs.build_jdk }} + cache: maven + + - name: Retrieve cached docker image + uses: actions/cache/restore@v3 + with: + key: druid-container-jdk${{ inputs.build_jdk }}.tar.gz + path: | + ./druid-container-jdk${{ inputs.build_jdk }}.tar.gz + ./integration-tests-ex/image/target/env.sh + + - name: Load docker image + run: | + docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz + docker images + + - name: Run IT + run: ${{ inputs.script }} diff --git a/.github/workflows/reusable-standard-its.yml b/.github/workflows/reusable-standard-its.yml new file mode 100644 index 00000000000..a2467d6c271 --- /dev/null +++ b/.github/workflows/reusable-standard-its.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Revised Integration Tests shared workflow +on: + workflow_call: + inputs: + runtime_jdk: + description: 'Which JDK version to use at runtime' + required: true + type: string + use_indexer: + description: 'Which indexer to use' + required: true + type: string + override_config_path: + description: 'Existing Druid configuration (using Docker) can be overridden or new config vars can be set e.g. when using cloud storage' + required: false + type: string + group: + required: true + type: string + description: 'Name of group of tests running (to display)' + testing_groups: + required: true + type: string + build_jdk: + description: 'Which jdk version was used to build' + required: true + type: string + mysql_driver: + description: 'MySQL driver to use' + required: false + type: string + default: com.mysql.jdbc.Driver + +env: + MVN: mvn --no-snapshot-updates + MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true + MAVEN_SKIP_TESTS: -P skip-tests + DOCKER_IP: 127.0.0.1 # for integration tests + MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + +jobs: + test: # Github job that runs a given standard/old IT for a given testing group + name: ${{ inputs.group }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }}) + runs-on: ubuntu-22.04 + steps: + - name: Checkout branch + uses: actions/checkout@v3 + + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ inputs.runtime_jdk }} + + - name: Restore Maven repository + id: maven-restore + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ inputs.build_jdk }}-${{ hashFiles('**/pom.xml') }} + + - name: Run IT + env: + MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} + MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }} + run: | + # Debug echo + echo "Mysql driver: ${MYSQL_DRIVER_CLASSNAME}" + echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}" + MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }} diff --git a/.github/workflows/reusable-unit-tests.yml b/.github/workflows/reusable-unit-tests.yml new file mode 100644 index 00000000000..1684a5fe570 --- /dev/null +++ b/.github/workflows/reusable-unit-tests.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +name: Revised Unit Tests shared workflow +on: + workflow_call: + inputs: + jdk: + required: true + type: string + description: 'JDK version used to test Druid' + sql_compatibility: + required: false + type: boolean + default: true + description: 'For SQL compatibility' + module: + required: true + type: string + description: 'Name of group of tests that are running' + maven_projects: + required: true + type: string + description: 'Name of group of tests running (to display)' + +env: + MVN: mvn -B + MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true + MAVEN_SKIP_TESTS: -P skip-tests + MAVEN_OPTS: -Xmx3000m + FORCE_COLOR: 2 + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + +jobs: + unit-tests: + name: ${{ inputs.module }} modules test + runs-on: ubuntu-latest + steps: + - name: checkout branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: setup jdk${{ inputs.jdk }} + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ inputs.jdk }} + + - name: Restore Maven repository + id: maven-restore + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml') }} + + - name: setup node + uses: actions/setup-node@v3 + with: + node-version: 16.17.0 + + - name: setup variables + run: | + export base_ref=${{ github.base_ref }} + echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV + export druid_use_default_value_for_null=${{ inputs.sql_compatibility }} + echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV + + - name: fetch base branch for test coverage + if: ${{ github.base_ref != '' }} + run: | + # Add merge target branch to determine diff. + # This is not needed for build triggered by tags, since there will be no code diff. + git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch + + - name: setup diff-test-coverage + run: npm install @connectis/diff-test-coverage@1.5.3 + + - name: test & coverage + env: + MAVEN_PROJECTS: ${{ inputs.maven_projects }} + MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }} + run: ./.github/scripts/unit_tests_script.sh diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml new file mode 100644 index 00000000000..056fa584e3d --- /dev/null +++ b/.github/workflows/revised-its.yml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Github workflow that runs revised/new ITs +on: + workflow_call + +jobs: + it: + strategy: + fail-fast: false + matrix: + #jdk: [8, 11, 17] + jdk: [8] + it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat] + #indexer: [indexer, middleManager] + indexer: [middleManager] + uses: apache/druid/.github/workflows/reusable-revised-its.yml@master + with: + build_jdk: ${{ matrix.jdk }} + runtime_jdk: ${{ matrix.jdk }} + use_indexer: ${{ matrix.indexer }} + script: ./it.sh github ${{ matrix.it }} + it: ${{ matrix.it }} diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml new file mode 100644 index 00000000000..c0e3a0b6f9a --- /dev/null +++ b/.github/workflows/standard-its.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Github workflow that runs standard/old ITs +on: + workflow_call + +jobs: + integration-index-tests-middleManager: + strategy: + fail-fast: false + matrix: + jdk: [8, 11] + testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction] + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: ${{ matrix.jdk }} + testing_groups: -Dgroups=${{ matrix.testing_group }} + use_indexer: middleManager + group: ${{ matrix.testing_group }} + + integration-index-tests-indexer: + strategy: + fail-fast: false + matrix: + jdk: [8, 11] + testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction] + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: ${{ matrix.jdk }} + testing_groups: -Dgroups=${{ matrix.testing_group }} + use_indexer: indexer + group: ${{ matrix.testing_group }} + + integration-query-tests-middleManager: + strategy: + fail-fast: false + matrix: + jdk: [8, 11] + testing_group: [query, query-retry, query-error, security, high-availability] + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: ${{ matrix.jdk }} + testing_groups: -Dgroups=${{ matrix.testing_group }} + use_indexer: middleManager + override_config_path: ./environment-configs/test-groups/prepopulated-data + group: ${{ matrix.testing_group }} + + integration-query-tests-middleManager-mariaDB: + strategy: + fail-fast: false + matrix: + jdk: [8, 11] + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: ${{ matrix.jdk }} + testing_groups: -Dgroups=query + use_indexer: middleManager + mysql_driver: org.mariadb.jdbc.Driver + override_config_path: ./environment-configs/test-groups/prepopulated-data + group: query + + integration-shuffle-deep-store-tests: + strategy: + fail-fast: false + matrix: + indexer: [indexer, middleManager] + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: 8 + testing_groups: -Dgroups=shuffle-deep-store + use_indexer: ${{ matrix.indexer }} + override_config_path: ./environment-configs/test-groups/shuffle-deep-store + group: shuffle deep store + + integration-custom-coordinator-duties-tests: + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: 8 + testing_groups: -Dgroups=custom-coordinator-duties + use_indexer: middleManager + override_config_path: ./environment-configs/test-groups/custom-coordinator-duties + group: custom coordinator duties + + integration-other-tests: + strategy: + fail-fast: false + matrix: + jdk: [8, 11] + indexer: [middleManager, indexer] + uses: apache/druid/.github/workflows/reusable-standard-its.yml@master + with: + build_jdk: 8 + runtime_jdk: ${{ matrix.jdk }} + testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties + use_indexer: ${{ matrix.indexer }} + group: other diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index f247bf01a19..a0baecdd0ee 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -37,6 +37,7 @@ env: jobs: static-checks: strategy: + fail-fast: false matrix: java: [ 'jdk8', 'jdk11', 'jdk17' ] runs-on: ubuntu-latest diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml new file mode 100644 index 00000000000..561ed0f0a47 --- /dev/null +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Unit & Integration tests CI +on: + push: + branches: + - master + - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches + pull_request: + branches: + - master + - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + +jobs: + build: + name: build (jdk${{ matrix.jdk }}) + strategy: + fail-fast: false + matrix: + jdk: [ '8', '11', '17' ] + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v3 + + - name: Cache Maven m2 repository + id: maven + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}-${{ matrix.jdk }} + + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ matrix.jdk }} + + - name: Maven build + id: maven_build + run: | + ./it.sh ci + + - name: Cache targets + id: target + uses: actions/cache@v3 + with: + path: | + ./**/target + key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}-${{ matrix.jdk }}-targets + + - name: Container build + run: | + ./it.sh image + source ./integration-tests-ex/image/target/env.sh + docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }} + + - name: Save docker container to archive + run: | + source ./integration-tests-ex/image/target/env.sh + echo $DRUID_IT_IMAGE_NAME + docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz + + - name: Cache image + id: docker_container + uses: actions/cache@v3 + with: + key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ hashFiles('**/pom.xml') }} + path: | + ./druid-container-jdk${{ matrix.jdk }}.tar.gz + ./integration-tests-ex/image/target/env.sh + restore-keys: | + druid-container-jdk${{ matrix.jdk }}.tar.gz- + + unit-tests: + strategy: + fail-fast: false + matrix: + sql_compatibility: [ false, true ] + name: unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }}) + uses: apache/druid/.github/workflows/unit-tests.yml@master + needs: build + with: + jdk: 8 + sql_compatibility: ${{ matrix.sql_compatibility }} + + unit-tests-phase2: + strategy: + fail-fast: false + matrix: + jdk: [11, 17] + sql_compatibility: [ false, true ] + name: unit tests (jdk${{ matrix.jdk }}, sql-compat=${{ matrix.sql_compatibility }}) + uses: apache/druid/.github/workflows/unit-tests.yml@master + needs: unit-tests + with: + jdk: ${{ matrix.jdk }} + sql_compatibility: ${{ matrix.sql_compatibility }} + + standard-its: + needs: unit-tests + uses: apache/druid/.github/workflows/standard-its.yml@master + + revised-its: + needs: unit-tests + uses: apache/druid/.github/workflows/revised-its.yml@master diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 77e0663adc1..ef500aa2efc 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,93 +11,50 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License -name: (openjdk8) Unit Tests CI on: - push: - branches: - - master - - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches - pull_request: - branches: - - master - - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches - -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -env: - MVN: mvn -B - MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true - MAVEN_SKIP_TESTS: -P skip-tests - MAVEN_OPTS: -Xmx3000m - FORCE_COLOR: 2 + workflow_call: + inputs: + jdk: + required: true + type: string + description: 'JDK version used to test Druid' + sql_compatibility: + required: false + type: boolean + default: true + description: 'For SQL compatibility' jobs: - unit-tests: - strategy: - matrix: - DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ] - runs-on: ubuntu-latest - steps: - - name: checkout branch - uses: actions/checkout@v3 - with: - fetch-depth: 0 + indexing_modules_test: + uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master + with: + jdk: ${{ inputs.jdk }} + sql_compatibility: ${{ inputs.sql_compatibility }} + module: indexing + maven_projects: 'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service' - - name: setup jdk8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 8 - cache: 'maven' + processing_modules_test: + uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master + with: + jdk: ${{ inputs.jdk }} + sql_compatibility: ${{ inputs.sql_compatibility }} + module: processing + maven_projects: 'processing' - - name: maven install - run: | - echo 'Running Maven install...' && - ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && - ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + server_modules_test: + uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master + with: + jdk: ${{ inputs.jdk }} + sql_compatibility: ${{ inputs.sql_compatibility }} + module: server + maven_projects: 'server' - - name: setup node - uses: actions/setup-node@v3 - with: - node-version: 16.17.0 - - - name: setup variables - run: | - export base_ref=${{ github.base_ref }} - echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV - export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }} - echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV - - - name: fetch base branch for test coverage - if: ${{ github.base_ref != '' }} - run: | - # Add merge target branch to determine diff. - # This is not needed for build triggered by tags, since there will be no code diff. - git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch - - - name: Setup diff-test-coverage - run: npm install @connectis/diff-test-coverage@1.5.3 - - - name: "(openjdk8) indexing modules test" - env: - MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service - run: ./.github/scripts/unit_tests_script.sh - - - name: "(openjdk8) processing module test" - env: - MAVEN_PROJECTS: processing - run: ./.github/scripts/unit_tests_script.sh - - - name: "(openjdk8) server module test" - env: - MAVEN_PROJECTS: server - run: ./.github/scripts/unit_tests_script.sh - - - name: "(openjdk8) other modules test" - env: - MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' - run: ./.github/scripts/unit_tests_script.sh + other_modules_test: + uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master + with: + jdk: ${{ inputs.jdk }} + sql_compatibility: ${{ inputs.sql_compatibility }} + module: other + maven_projects: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' diff --git a/integration-tests/docker/docker-compose.query-error-test.yml b/integration-tests/docker/docker-compose.query-error-test.yml index 5dc175829ad..f75ff2efad7 100644 --- a/integration-tests/docker/docker-compose.query-error-test.yml +++ b/integration-tests/docker/docker-compose.query-error-test.yml @@ -78,7 +78,7 @@ services: druid-it-net: ipv4_address: 172.172.172.14 ports: - - 8084:8083 + - 8086:8083 #8084 is used by mono service on GHA runners - 8284:8283 - 5010:5007 privileged: true diff --git a/integration-tests/docker/docker-compose.query-retry-test.yml b/integration-tests/docker/docker-compose.query-retry-test.yml index adba343557e..2efb859a6f8 100644 --- a/integration-tests/docker/docker-compose.query-retry-test.yml +++ b/integration-tests/docker/docker-compose.query-retry-test.yml @@ -78,7 +78,7 @@ services: druid-it-net: ipv4_address: 172.172.172.14 ports: - - 8084:8083 + - 8086:8083 #8084 is used by mono service on GHA runners - 8284:8283 - 5010:5007 privileged: true diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java index fe95c27bc7c..72e682c9fbf 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java @@ -108,6 +108,8 @@ public class ITJdbcQueryTest }; // ensure that wikipedia segments are loaded completely dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE); + dataLoaderHelper.waitUntilDatasourceIsReady("wikipedia"); + dataLoaderHelper.waitUntilDatasourceIsReady("twitterstream"); } @Test diff --git a/it.sh b/it.sh index 9988f6b5c80..7fb3889b559 100755 --- a/it.sh +++ b/it.sh @@ -31,6 +31,8 @@ function usage { cat < run one IT in Travis (build dist, image, run test, tail logs) + github + run one IT in Github Workflows (run test, tail logs) prune prune Docker volumes @@ -189,6 +193,9 @@ case $CMD in "help" ) usage ;; + "ci" ) + mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE -T1.0C + ;; "build" ) mvn clean package -P dist $MAVEN_IGNORE -T1.0C ;; @@ -230,6 +237,11 @@ case $CMD in $0 test $CATEGORY $0 tail $CATEGORY ;; + "github" ) + prepare_category $1 + $0 test $CATEGORY + $0 tail $CATEGORY + ;; "prune" ) # Caution: this removes all volumes, which is generally what you # want when testing.