mirror of https://github.com/apache/druid.git
Migrate ITs from Travis to GHA (#13681)
This commit is contained in:
Tejaswini Bandlamudi
GitHub
parent
7a3bd89a85
commit
c95a26cae3
|
|
@ -0,0 +1,73 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Revised Integration Tests shared workflow
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
script:
|
||||
description: 'Which IT script to run'
|
||||
required: true
|
||||
type: string
|
||||
build_jdk:
|
||||
description: 'Which jdk version was used to build'
|
||||
required: true
|
||||
type: string
|
||||
runtime_jdk:
|
||||
description: 'Which JDK version to use at runtime'
|
||||
required: true
|
||||
type: string
|
||||
use_indexer:
|
||||
description: 'Which indexer to use'
|
||||
required: true
|
||||
type: string
|
||||
it:
|
||||
description: 'IT test Category'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
env:
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||
|
||||
jobs:
|
||||
test: # Github job that runs a given revised/new IT against retrieved cached druid docker image
|
||||
name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }})
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Checkout branch
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Setup java
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: 'zulu'
|
||||
java-version: ${{ inputs.build_jdk }}
|
||||
cache: maven
|
||||
|
||||
- name: Retrieve cached docker image
|
||||
uses: actions/cache/restore@v3
|
||||
with:
|
||||
key: druid-container-jdk${{ inputs.build_jdk }}.tar.gz
|
||||
path: |
|
||||
./druid-container-jdk${{ inputs.build_jdk }}.tar.gz
|
||||
./integration-tests-ex/image/target/env.sh
|
||||
|
||||
- name: Load docker image
|
||||
run: |
|
||||
docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz
|
||||
docker images
|
||||
|
||||
- name: Run IT
|
||||
run: ${{ inputs.script }}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Revised Integration Tests shared workflow
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runtime_jdk:
|
||||
description: 'Which JDK version to use at runtime'
|
||||
required: true
|
||||
type: string
|
||||
use_indexer:
|
||||
description: 'Which indexer to use'
|
||||
required: true
|
||||
type: string
|
||||
override_config_path:
|
||||
description: 'Existing Druid configuration (using Docker) can be overridden or new config vars can be set e.g. when using cloud storage'
|
||||
required: false
|
||||
type: string
|
||||
group:
|
||||
required: true
|
||||
type: string
|
||||
description: 'Name of group of tests running (to display)'
|
||||
testing_groups:
|
||||
required: true
|
||||
type: string
|
||||
build_jdk:
|
||||
description: 'Which jdk version was used to build'
|
||||
required: true
|
||||
type: string
|
||||
mysql_driver:
|
||||
description: 'MySQL driver to use'
|
||||
required: false
|
||||
type: string
|
||||
default: com.mysql.jdbc.Driver
|
||||
|
||||
env:
|
||||
MVN: mvn --no-snapshot-updates
|
||||
MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
|
||||
MAVEN_SKIP_TESTS: -P skip-tests
|
||||
DOCKER_IP: 127.0.0.1 # for integration tests
|
||||
MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||
|
||||
jobs:
|
||||
test: # Github job that runs a given standard/old IT for a given testing group
|
||||
name: ${{ inputs.group }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Checkout branch
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Setup java
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: 'zulu'
|
||||
java-version: ${{ inputs.runtime_jdk }}
|
||||
|
||||
- name: Restore Maven repository
|
||||
id: maven-restore
|
||||
uses: actions/cache/restore@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: maven-${{ runner.os }}-${{ inputs.build_jdk }}-${{ hashFiles('**/pom.xml') }}
|
||||
|
||||
- name: Run IT
|
||||
env:
|
||||
MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
|
||||
MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
|
||||
run: |
|
||||
# Debug echo
|
||||
echo "Mysql driver: ${MYSQL_DRIVER_CLASSNAME}"
|
||||
echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
|
||||
MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License
|
||||
|
||||
name: Revised Unit Tests shared workflow
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
jdk:
|
||||
required: true
|
||||
type: string
|
||||
description: 'JDK version used to test Druid'
|
||||
sql_compatibility:
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
description: 'For SQL compatibility'
|
||||
module:
|
||||
required: true
|
||||
type: string
|
||||
description: 'Name of group of tests that are running'
|
||||
maven_projects:
|
||||
required: true
|
||||
type: string
|
||||
description: 'Name of group of tests running (to display)'
|
||||
|
||||
env:
|
||||
MVN: mvn -B
|
||||
MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
|
||||
MAVEN_SKIP_TESTS: -P skip-tests
|
||||
MAVEN_OPTS: -Xmx3000m
|
||||
FORCE_COLOR: 2
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||
|
||||
jobs:
|
||||
unit-tests:
|
||||
name: ${{ inputs.module }} modules test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout branch
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup jdk${{ inputs.jdk }}
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: 'zulu'
|
||||
java-version: ${{ inputs.jdk }}
|
||||
|
||||
- name: Restore Maven repository
|
||||
id: maven-restore
|
||||
uses: actions/cache/restore@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: maven-${{ runner.os }}-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml') }}
|
||||
|
||||
- name: setup node
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 16.17.0
|
||||
|
||||
- name: setup variables
|
||||
run: |
|
||||
export base_ref=${{ github.base_ref }}
|
||||
echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
|
||||
export druid_use_default_value_for_null=${{ inputs.sql_compatibility }}
|
||||
echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
|
||||
|
||||
- name: fetch base branch for test coverage
|
||||
if: ${{ github.base_ref != '' }}
|
||||
run: |
|
||||
# Add merge target branch to determine diff.
|
||||
# This is not needed for build triggered by tags, since there will be no code diff.
|
||||
git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
|
||||
|
||||
- name: setup diff-test-coverage
|
||||
run: npm install @connectis/diff-test-coverage@1.5.3
|
||||
|
||||
- name: test & coverage
|
||||
env:
|
||||
MAVEN_PROJECTS: ${{ inputs.maven_projects }}
|
||||
MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
|
||||
run: ./.github/scripts/unit_tests_script.sh
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Github workflow that runs revised/new ITs
|
||||
on:
|
||||
workflow_call
|
||||
|
||||
jobs:
|
||||
it:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
#jdk: [8, 11, 17]
|
||||
jdk: [8]
|
||||
it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat]
|
||||
#indexer: [indexer, middleManager]
|
||||
indexer: [middleManager]
|
||||
uses: apache/druid/.github/workflows/reusable-revised-its.yml@master
|
||||
with:
|
||||
build_jdk: ${{ matrix.jdk }}
|
||||
runtime_jdk: ${{ matrix.jdk }}
|
||||
use_indexer: ${{ matrix.indexer }}
|
||||
script: ./it.sh github ${{ matrix.it }}
|
||||
it: ${{ matrix.it }}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Github workflow that runs standard/old ITs
|
||||
on:
|
||||
workflow_call
|
||||
|
||||
jobs:
|
||||
integration-index-tests-middleManager:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [8, 11]
|
||||
testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: ${{ matrix.jdk }}
|
||||
testing_groups: -Dgroups=${{ matrix.testing_group }}
|
||||
use_indexer: middleManager
|
||||
group: ${{ matrix.testing_group }}
|
||||
|
||||
integration-index-tests-indexer:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [8, 11]
|
||||
testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: ${{ matrix.jdk }}
|
||||
testing_groups: -Dgroups=${{ matrix.testing_group }}
|
||||
use_indexer: indexer
|
||||
group: ${{ matrix.testing_group }}
|
||||
|
||||
integration-query-tests-middleManager:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [8, 11]
|
||||
testing_group: [query, query-retry, query-error, security, high-availability]
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: ${{ matrix.jdk }}
|
||||
testing_groups: -Dgroups=${{ matrix.testing_group }}
|
||||
use_indexer: middleManager
|
||||
override_config_path: ./environment-configs/test-groups/prepopulated-data
|
||||
group: ${{ matrix.testing_group }}
|
||||
|
||||
integration-query-tests-middleManager-mariaDB:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [8, 11]
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: ${{ matrix.jdk }}
|
||||
testing_groups: -Dgroups=query
|
||||
use_indexer: middleManager
|
||||
mysql_driver: org.mariadb.jdbc.Driver
|
||||
override_config_path: ./environment-configs/test-groups/prepopulated-data
|
||||
group: query
|
||||
|
||||
integration-shuffle-deep-store-tests:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
indexer: [indexer, middleManager]
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: 8
|
||||
testing_groups: -Dgroups=shuffle-deep-store
|
||||
use_indexer: ${{ matrix.indexer }}
|
||||
override_config_path: ./environment-configs/test-groups/shuffle-deep-store
|
||||
group: shuffle deep store
|
||||
|
||||
integration-custom-coordinator-duties-tests:
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: 8
|
||||
testing_groups: -Dgroups=custom-coordinator-duties
|
||||
use_indexer: middleManager
|
||||
override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
|
||||
group: custom coordinator duties
|
||||
|
||||
integration-other-tests:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [8, 11]
|
||||
indexer: [middleManager, indexer]
|
||||
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: ${{ matrix.jdk }}
|
||||
testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
|
||||
use_indexer: ${{ matrix.indexer }}
|
||||
group: other
|
||||
|
|
@ -37,6 +37,7 @@ env:
|
|||
jobs:
|
||||
static-checks:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
java: [ 'jdk8', 'jdk11', 'jdk17' ]
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Unit & Integration tests CI
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
||||
|
||||
concurrency:
|
||||
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build (jdk${{ matrix.jdk }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [ '8', '11', '17' ]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout branch
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Cache Maven m2 repository
|
||||
id: maven
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
maven-${{ runner.os }}-${{ matrix.jdk }}
|
||||
|
||||
- name: Setup java
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
distribution: 'zulu'
|
||||
java-version: ${{ matrix.jdk }}
|
||||
|
||||
- name: Maven build
|
||||
id: maven_build
|
||||
run: |
|
||||
./it.sh ci
|
||||
|
||||
- name: Cache targets
|
||||
id: target
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
./**/target
|
||||
key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
maven-${{ runner.os }}-${{ matrix.jdk }}-targets
|
||||
|
||||
- name: Container build
|
||||
run: |
|
||||
./it.sh image
|
||||
source ./integration-tests-ex/image/target/env.sh
|
||||
docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }}
|
||||
|
||||
- name: Save docker container to archive
|
||||
run: |
|
||||
source ./integration-tests-ex/image/target/env.sh
|
||||
echo $DRUID_IT_IMAGE_NAME
|
||||
docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz
|
||||
|
||||
- name: Cache image
|
||||
id: docker_container
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ hashFiles('**/pom.xml') }}
|
||||
path: |
|
||||
./druid-container-jdk${{ matrix.jdk }}.tar.gz
|
||||
./integration-tests-ex/image/target/env.sh
|
||||
restore-keys: |
|
||||
druid-container-jdk${{ matrix.jdk }}.tar.gz-
|
||||
|
||||
unit-tests:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
sql_compatibility: [ false, true ]
|
||||
name: unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})
|
||||
uses: apache/druid/.github/workflows/unit-tests.yml@master
|
||||
needs: build
|
||||
with:
|
||||
jdk: 8
|
||||
sql_compatibility: ${{ matrix.sql_compatibility }}
|
||||
|
||||
unit-tests-phase2:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk: [11, 17]
|
||||
sql_compatibility: [ false, true ]
|
||||
name: unit tests (jdk${{ matrix.jdk }}, sql-compat=${{ matrix.sql_compatibility }})
|
||||
uses: apache/druid/.github/workflows/unit-tests.yml@master
|
||||
needs: unit-tests
|
||||
with:
|
||||
jdk: ${{ matrix.jdk }}
|
||||
sql_compatibility: ${{ matrix.sql_compatibility }}
|
||||
|
||||
standard-its:
|
||||
needs: unit-tests
|
||||
uses: apache/druid/.github/workflows/standard-its.yml@master
|
||||
|
||||
revised-its:
|
||||
needs: unit-tests
|
||||
uses: apache/druid/.github/workflows/revised-its.yml@master
|
||||
|
|
@ -11,93 +11,50 @@
|
|||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# limitations under the License
|
||||
|
||||
name: (openjdk8) Unit Tests CI
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
||||
|
||||
concurrency:
|
||||
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
MVN: mvn -B
|
||||
MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
|
||||
MAVEN_SKIP_TESTS: -P skip-tests
|
||||
MAVEN_OPTS: -Xmx3000m
|
||||
FORCE_COLOR: 2
|
||||
workflow_call:
|
||||
inputs:
|
||||
jdk:
|
||||
required: true
|
||||
type: string
|
||||
description: 'JDK version used to test Druid'
|
||||
sql_compatibility:
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
description: 'For SQL compatibility'
|
||||
|
||||
jobs:
|
||||
unit-tests:
|
||||
strategy:
|
||||
matrix:
|
||||
DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout branch
|
||||
uses: actions/checkout@v3
|
||||
indexing_modules_test:
|
||||
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||
with:
|
||||
fetch-depth: 0
|
||||
jdk: ${{ inputs.jdk }}
|
||||
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||
module: indexing
|
||||
maven_projects: 'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
|
||||
|
||||
- name: setup jdk8
|
||||
uses: actions/setup-java@v3
|
||||
processing_modules_test:
|
||||
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||
with:
|
||||
distribution: 'zulu'
|
||||
java-version: 8
|
||||
cache: 'maven'
|
||||
jdk: ${{ inputs.jdk }}
|
||||
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||
module: processing
|
||||
maven_projects: 'processing'
|
||||
|
||||
- name: maven install
|
||||
run: |
|
||||
echo 'Running Maven install...' &&
|
||||
${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
|
||||
${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
|
||||
|
||||
- name: setup node
|
||||
uses: actions/setup-node@v3
|
||||
server_modules_test:
|
||||
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||
with:
|
||||
node-version: 16.17.0
|
||||
jdk: ${{ inputs.jdk }}
|
||||
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||
module: server
|
||||
maven_projects: 'server'
|
||||
|
||||
- name: setup variables
|
||||
run: |
|
||||
export base_ref=${{ github.base_ref }}
|
||||
echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
|
||||
export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
|
||||
echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
|
||||
|
||||
- name: fetch base branch for test coverage
|
||||
if: ${{ github.base_ref != '' }}
|
||||
run: |
|
||||
# Add merge target branch to determine diff.
|
||||
# This is not needed for build triggered by tags, since there will be no code diff.
|
||||
git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
|
||||
|
||||
- name: Setup diff-test-coverage
|
||||
run: npm install @connectis/diff-test-coverage@1.5.3
|
||||
|
||||
- name: "(openjdk8) indexing modules test"
|
||||
env:
|
||||
MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
|
||||
run: ./.github/scripts/unit_tests_script.sh
|
||||
|
||||
- name: "(openjdk8) processing module test"
|
||||
env:
|
||||
MAVEN_PROJECTS: processing
|
||||
run: ./.github/scripts/unit_tests_script.sh
|
||||
|
||||
- name: "(openjdk8) server module test"
|
||||
env:
|
||||
MAVEN_PROJECTS: server
|
||||
run: ./.github/scripts/unit_tests_script.sh
|
||||
|
||||
- name: "(openjdk8) other modules test"
|
||||
env:
|
||||
MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
|
||||
run: ./.github/scripts/unit_tests_script.sh
|
||||
other_modules_test:
|
||||
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||
with:
|
||||
jdk: ${{ inputs.jdk }}
|
||||
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||
module: other
|
||||
maven_projects: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ services:
|
|||
druid-it-net:
|
||||
ipv4_address: 172.172.172.14
|
||||
ports:
|
||||
- 8084:8083
|
||||
- 8086:8083 #8084 is used by mono service on GHA runners
|
||||
- 8284:8283
|
||||
- 5010:5007
|
||||
privileged: true
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ services:
|
|||
druid-it-net:
|
||||
ipv4_address: 172.172.172.14
|
||||
ports:
|
||||
- 8084:8083
|
||||
- 8086:8083 #8084 is used by mono service on GHA runners
|
||||
- 8284:8283
|
||||
- 5010:5007
|
||||
privileged: true
|
||||
|
|
|
|||
|
|
@ -108,6 +108,8 @@ public class ITJdbcQueryTest
|
|||
};
|
||||
// ensure that wikipedia segments are loaded completely
|
||||
dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
|
||||
dataLoaderHelper.waitUntilDatasourceIsReady("wikipedia");
|
||||
dataLoaderHelper.waitUntilDatasourceIsReady("twitterstream");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
12
it.sh
12
it.sh
|
|
@ -31,6 +31,8 @@ function usage
|
|||
{
|
||||
cat <<EOF
|
||||
Usage: $0 cmd [category]
|
||||
ci
|
||||
build Druid and the distribution for CI pipelines
|
||||
build
|
||||
build Druid and the distribution
|
||||
dist
|
||||
|
|
@ -49,6 +51,8 @@ Usage: $0 cmd [category]
|
|||
show the last 20 lines of each container log
|
||||
travis <category>
|
||||
run one IT in Travis (build dist, image, run test, tail logs)
|
||||
github <category>
|
||||
run one IT in Github Workflows (run test, tail logs)
|
||||
prune
|
||||
prune Docker volumes
|
||||
|
||||
|
|
@ -189,6 +193,9 @@ case $CMD in
|
|||
"help" )
|
||||
usage
|
||||
;;
|
||||
"ci" )
|
||||
mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE -T1.0C
|
||||
;;
|
||||
"build" )
|
||||
mvn clean package -P dist $MAVEN_IGNORE -T1.0C
|
||||
;;
|
||||
|
|
@ -230,6 +237,11 @@ case $CMD in
|
|||
$0 test $CATEGORY
|
||||
$0 tail $CATEGORY
|
||||
;;
|
||||
"github" )
|
||||
prepare_category $1
|
||||
$0 test $CATEGORY
|
||||
$0 tail $CATEGORY
|
||||
;;
|
||||
"prune" )
|
||||
# Caution: this removes all volumes, which is generally what you
|
||||
# want when testing.
|
||||
|
|
|
|||
Loading…
Reference in New Issue