mirror of https://github.com/apache/druid.git
Migrate ITs from Travis to GHA (#13681)
This commit is contained in:
parent
7a3bd89a85
commit
c95a26cae3
|
@ -0,0 +1,73 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
name: Revised Integration Tests shared workflow
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
script:
|
||||||
|
description: 'Which IT script to run'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
build_jdk:
|
||||||
|
description: 'Which jdk version was used to build'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
runtime_jdk:
|
||||||
|
description: 'Which JDK version to use at runtime'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
use_indexer:
|
||||||
|
description: 'Which indexer to use'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
it:
|
||||||
|
description: 'IT test Category'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
env:
|
||||||
|
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test: # Github job that runs a given revised/new IT against retrieved cached druid docker image
|
||||||
|
name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }})
|
||||||
|
runs-on: ubuntu-22.04
|
||||||
|
steps:
|
||||||
|
- name: Checkout branch
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Setup java
|
||||||
|
uses: actions/setup-java@v3
|
||||||
|
with:
|
||||||
|
distribution: 'zulu'
|
||||||
|
java-version: ${{ inputs.build_jdk }}
|
||||||
|
cache: maven
|
||||||
|
|
||||||
|
- name: Retrieve cached docker image
|
||||||
|
uses: actions/cache/restore@v3
|
||||||
|
with:
|
||||||
|
key: druid-container-jdk${{ inputs.build_jdk }}.tar.gz
|
||||||
|
path: |
|
||||||
|
./druid-container-jdk${{ inputs.build_jdk }}.tar.gz
|
||||||
|
./integration-tests-ex/image/target/env.sh
|
||||||
|
|
||||||
|
- name: Load docker image
|
||||||
|
run: |
|
||||||
|
docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz
|
||||||
|
docker images
|
||||||
|
|
||||||
|
- name: Run IT
|
||||||
|
run: ${{ inputs.script }}
|
|
@ -0,0 +1,86 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
name: Revised Integration Tests shared workflow
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
runtime_jdk:
|
||||||
|
description: 'Which JDK version to use at runtime'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
use_indexer:
|
||||||
|
description: 'Which indexer to use'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
override_config_path:
|
||||||
|
description: 'Existing Druid configuration (using Docker) can be overridden or new config vars can be set e.g. when using cloud storage'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
group:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'Name of group of tests running (to display)'
|
||||||
|
testing_groups:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
build_jdk:
|
||||||
|
description: 'Which jdk version was used to build'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
mysql_driver:
|
||||||
|
description: 'MySQL driver to use'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: com.mysql.jdbc.Driver
|
||||||
|
|
||||||
|
env:
|
||||||
|
MVN: mvn --no-snapshot-updates
|
||||||
|
MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
|
||||||
|
MAVEN_SKIP_TESTS: -P skip-tests
|
||||||
|
DOCKER_IP: 127.0.0.1 # for integration tests
|
||||||
|
MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
|
||||||
|
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test: # Github job that runs a given standard/old IT for a given testing group
|
||||||
|
name: ${{ inputs.group }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
|
||||||
|
runs-on: ubuntu-22.04
|
||||||
|
steps:
|
||||||
|
- name: Checkout branch
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Setup java
|
||||||
|
uses: actions/setup-java@v3
|
||||||
|
with:
|
||||||
|
distribution: 'zulu'
|
||||||
|
java-version: ${{ inputs.runtime_jdk }}
|
||||||
|
|
||||||
|
- name: Restore Maven repository
|
||||||
|
id: maven-restore
|
||||||
|
uses: actions/cache/restore@v3
|
||||||
|
with:
|
||||||
|
path: ~/.m2/repository
|
||||||
|
key: maven-${{ runner.os }}-${{ inputs.build_jdk }}-${{ hashFiles('**/pom.xml') }}
|
||||||
|
|
||||||
|
- name: Run IT
|
||||||
|
env:
|
||||||
|
MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
|
||||||
|
MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
|
||||||
|
run: |
|
||||||
|
# Debug echo
|
||||||
|
echo "Mysql driver: ${MYSQL_DRIVER_CLASSNAME}"
|
||||||
|
echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
|
||||||
|
MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}
|
|
@ -0,0 +1,95 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License
|
||||||
|
|
||||||
|
name: Revised Unit Tests shared workflow
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
jdk:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'JDK version used to test Druid'
|
||||||
|
sql_compatibility:
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: true
|
||||||
|
description: 'For SQL compatibility'
|
||||||
|
module:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'Name of group of tests that are running'
|
||||||
|
maven_projects:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'Name of group of tests running (to display)'
|
||||||
|
|
||||||
|
env:
|
||||||
|
MVN: mvn -B
|
||||||
|
MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
|
||||||
|
MAVEN_SKIP_TESTS: -P skip-tests
|
||||||
|
MAVEN_OPTS: -Xmx3000m
|
||||||
|
FORCE_COLOR: 2
|
||||||
|
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
unit-tests:
|
||||||
|
name: ${{ inputs.module }} modules test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: checkout branch
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: setup jdk${{ inputs.jdk }}
|
||||||
|
uses: actions/setup-java@v3
|
||||||
|
with:
|
||||||
|
distribution: 'zulu'
|
||||||
|
java-version: ${{ inputs.jdk }}
|
||||||
|
|
||||||
|
- name: Restore Maven repository
|
||||||
|
id: maven-restore
|
||||||
|
uses: actions/cache/restore@v3
|
||||||
|
with:
|
||||||
|
path: ~/.m2/repository
|
||||||
|
key: maven-${{ runner.os }}-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml') }}
|
||||||
|
|
||||||
|
- name: setup node
|
||||||
|
uses: actions/setup-node@v3
|
||||||
|
with:
|
||||||
|
node-version: 16.17.0
|
||||||
|
|
||||||
|
- name: setup variables
|
||||||
|
run: |
|
||||||
|
export base_ref=${{ github.base_ref }}
|
||||||
|
echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
|
||||||
|
export druid_use_default_value_for_null=${{ inputs.sql_compatibility }}
|
||||||
|
echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: fetch base branch for test coverage
|
||||||
|
if: ${{ github.base_ref != '' }}
|
||||||
|
run: |
|
||||||
|
# Add merge target branch to determine diff.
|
||||||
|
# This is not needed for build triggered by tags, since there will be no code diff.
|
||||||
|
git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
|
||||||
|
|
||||||
|
- name: setup diff-test-coverage
|
||||||
|
run: npm install @connectis/diff-test-coverage@1.5.3
|
||||||
|
|
||||||
|
- name: test & coverage
|
||||||
|
env:
|
||||||
|
MAVEN_PROJECTS: ${{ inputs.maven_projects }}
|
||||||
|
MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
|
||||||
|
run: ./.github/scripts/unit_tests_script.sh
|
|
@ -0,0 +1,36 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Github workflow that runs revised/new ITs
|
||||||
|
on:
|
||||||
|
workflow_call
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
it:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
#jdk: [8, 11, 17]
|
||||||
|
jdk: [8]
|
||||||
|
it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat]
|
||||||
|
#indexer: [indexer, middleManager]
|
||||||
|
indexer: [middleManager]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-revised-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: ${{ matrix.jdk }}
|
||||||
|
runtime_jdk: ${{ matrix.jdk }}
|
||||||
|
use_indexer: ${{ matrix.indexer }}
|
||||||
|
script: ./it.sh github ${{ matrix.it }}
|
||||||
|
it: ${{ matrix.it }}
|
|
@ -0,0 +1,115 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# Github workflow that runs standard/old ITs
|
||||||
|
on:
|
||||||
|
workflow_call
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
integration-index-tests-middleManager:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [8, 11]
|
||||||
|
testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: ${{ matrix.jdk }}
|
||||||
|
testing_groups: -Dgroups=${{ matrix.testing_group }}
|
||||||
|
use_indexer: middleManager
|
||||||
|
group: ${{ matrix.testing_group }}
|
||||||
|
|
||||||
|
integration-index-tests-indexer:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [8, 11]
|
||||||
|
testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: ${{ matrix.jdk }}
|
||||||
|
testing_groups: -Dgroups=${{ matrix.testing_group }}
|
||||||
|
use_indexer: indexer
|
||||||
|
group: ${{ matrix.testing_group }}
|
||||||
|
|
||||||
|
integration-query-tests-middleManager:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [8, 11]
|
||||||
|
testing_group: [query, query-retry, query-error, security, high-availability]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: ${{ matrix.jdk }}
|
||||||
|
testing_groups: -Dgroups=${{ matrix.testing_group }}
|
||||||
|
use_indexer: middleManager
|
||||||
|
override_config_path: ./environment-configs/test-groups/prepopulated-data
|
||||||
|
group: ${{ matrix.testing_group }}
|
||||||
|
|
||||||
|
integration-query-tests-middleManager-mariaDB:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [8, 11]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: ${{ matrix.jdk }}
|
||||||
|
testing_groups: -Dgroups=query
|
||||||
|
use_indexer: middleManager
|
||||||
|
mysql_driver: org.mariadb.jdbc.Driver
|
||||||
|
override_config_path: ./environment-configs/test-groups/prepopulated-data
|
||||||
|
group: query
|
||||||
|
|
||||||
|
integration-shuffle-deep-store-tests:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
indexer: [indexer, middleManager]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: 8
|
||||||
|
testing_groups: -Dgroups=shuffle-deep-store
|
||||||
|
use_indexer: ${{ matrix.indexer }}
|
||||||
|
override_config_path: ./environment-configs/test-groups/shuffle-deep-store
|
||||||
|
group: shuffle deep store
|
||||||
|
|
||||||
|
integration-custom-coordinator-duties-tests:
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: 8
|
||||||
|
testing_groups: -Dgroups=custom-coordinator-duties
|
||||||
|
use_indexer: middleManager
|
||||||
|
override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
|
||||||
|
group: custom coordinator duties
|
||||||
|
|
||||||
|
integration-other-tests:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [8, 11]
|
||||||
|
indexer: [middleManager, indexer]
|
||||||
|
uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
|
||||||
|
with:
|
||||||
|
build_jdk: 8
|
||||||
|
runtime_jdk: ${{ matrix.jdk }}
|
||||||
|
testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
|
||||||
|
use_indexer: ${{ matrix.indexer }}
|
||||||
|
group: other
|
|
@ -37,6 +37,7 @@ env:
|
||||||
jobs:
|
jobs:
|
||||||
static-checks:
|
static-checks:
|
||||||
strategy:
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
java: [ 'jdk8', 'jdk11', 'jdk17' ]
|
java: [ 'jdk8', 'jdk11', 'jdk17' ]
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
|
@ -0,0 +1,130 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
name: Unit & Integration tests CI
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: build (jdk${{ matrix.jdk }})
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [ '8', '11', '17' ]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout branch
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Cache Maven m2 repository
|
||||||
|
id: maven
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: ~/.m2/repository
|
||||||
|
key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ hashFiles('**/pom.xml') }}
|
||||||
|
restore-keys: |
|
||||||
|
maven-${{ runner.os }}-${{ matrix.jdk }}
|
||||||
|
|
||||||
|
- name: Setup java
|
||||||
|
uses: actions/setup-java@v3
|
||||||
|
with:
|
||||||
|
distribution: 'zulu'
|
||||||
|
java-version: ${{ matrix.jdk }}
|
||||||
|
|
||||||
|
- name: Maven build
|
||||||
|
id: maven_build
|
||||||
|
run: |
|
||||||
|
./it.sh ci
|
||||||
|
|
||||||
|
- name: Cache targets
|
||||||
|
id: target
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
./**/target
|
||||||
|
key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ hashFiles('**/pom.xml') }}
|
||||||
|
restore-keys: |
|
||||||
|
maven-${{ runner.os }}-${{ matrix.jdk }}-targets
|
||||||
|
|
||||||
|
- name: Container build
|
||||||
|
run: |
|
||||||
|
./it.sh image
|
||||||
|
source ./integration-tests-ex/image/target/env.sh
|
||||||
|
docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }}
|
||||||
|
|
||||||
|
- name: Save docker container to archive
|
||||||
|
run: |
|
||||||
|
source ./integration-tests-ex/image/target/env.sh
|
||||||
|
echo $DRUID_IT_IMAGE_NAME
|
||||||
|
docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz
|
||||||
|
|
||||||
|
- name: Cache image
|
||||||
|
id: docker_container
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ hashFiles('**/pom.xml') }}
|
||||||
|
path: |
|
||||||
|
./druid-container-jdk${{ matrix.jdk }}.tar.gz
|
||||||
|
./integration-tests-ex/image/target/env.sh
|
||||||
|
restore-keys: |
|
||||||
|
druid-container-jdk${{ matrix.jdk }}.tar.gz-
|
||||||
|
|
||||||
|
unit-tests:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
sql_compatibility: [ false, true ]
|
||||||
|
name: unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})
|
||||||
|
uses: apache/druid/.github/workflows/unit-tests.yml@master
|
||||||
|
needs: build
|
||||||
|
with:
|
||||||
|
jdk: 8
|
||||||
|
sql_compatibility: ${{ matrix.sql_compatibility }}
|
||||||
|
|
||||||
|
unit-tests-phase2:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
jdk: [11, 17]
|
||||||
|
sql_compatibility: [ false, true ]
|
||||||
|
name: unit tests (jdk${{ matrix.jdk }}, sql-compat=${{ matrix.sql_compatibility }})
|
||||||
|
uses: apache/druid/.github/workflows/unit-tests.yml@master
|
||||||
|
needs: unit-tests
|
||||||
|
with:
|
||||||
|
jdk: ${{ matrix.jdk }}
|
||||||
|
sql_compatibility: ${{ matrix.sql_compatibility }}
|
||||||
|
|
||||||
|
standard-its:
|
||||||
|
needs: unit-tests
|
||||||
|
uses: apache/druid/.github/workflows/standard-its.yml@master
|
||||||
|
|
||||||
|
revised-its:
|
||||||
|
needs: unit-tests
|
||||||
|
uses: apache/druid/.github/workflows/revised-its.yml@master
|
|
@ -11,93 +11,50 @@
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License
|
||||||
|
|
||||||
name: (openjdk8) Unit Tests CI
|
|
||||||
on:
|
on:
|
||||||
push:
|
workflow_call:
|
||||||
branches:
|
inputs:
|
||||||
- master
|
jdk:
|
||||||
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
required: true
|
||||||
pull_request:
|
type: string
|
||||||
branches:
|
description: 'JDK version used to test Druid'
|
||||||
- master
|
sql_compatibility:
|
||||||
- /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
|
required: false
|
||||||
|
type: boolean
|
||||||
concurrency:
|
default: true
|
||||||
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
|
description: 'For SQL compatibility'
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
env:
|
|
||||||
MVN: mvn -B
|
|
||||||
MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
|
|
||||||
MAVEN_SKIP_TESTS: -P skip-tests
|
|
||||||
MAVEN_OPTS: -Xmx3000m
|
|
||||||
FORCE_COLOR: 2
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
unit-tests:
|
indexing_modules_test:
|
||||||
strategy:
|
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||||
matrix:
|
|
||||||
DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: checkout branch
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
jdk: ${{ inputs.jdk }}
|
||||||
|
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||||
|
module: indexing
|
||||||
|
maven_projects: 'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
|
||||||
|
|
||||||
- name: setup jdk8
|
processing_modules_test:
|
||||||
uses: actions/setup-java@v3
|
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||||
with:
|
with:
|
||||||
distribution: 'zulu'
|
jdk: ${{ inputs.jdk }}
|
||||||
java-version: 8
|
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||||
cache: 'maven'
|
module: processing
|
||||||
|
maven_projects: 'processing'
|
||||||
|
|
||||||
- name: maven install
|
server_modules_test:
|
||||||
run: |
|
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||||
echo 'Running Maven install...' &&
|
|
||||||
${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
|
|
||||||
${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
|
|
||||||
|
|
||||||
- name: setup node
|
|
||||||
uses: actions/setup-node@v3
|
|
||||||
with:
|
with:
|
||||||
node-version: 16.17.0
|
jdk: ${{ inputs.jdk }}
|
||||||
|
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||||
|
module: server
|
||||||
|
maven_projects: 'server'
|
||||||
|
|
||||||
- name: setup variables
|
other_modules_test:
|
||||||
run: |
|
uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
|
||||||
export base_ref=${{ github.base_ref }}
|
with:
|
||||||
echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
|
jdk: ${{ inputs.jdk }}
|
||||||
export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
|
sql_compatibility: ${{ inputs.sql_compatibility }}
|
||||||
echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
|
module: other
|
||||||
|
maven_projects: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
|
||||||
- name: fetch base branch for test coverage
|
|
||||||
if: ${{ github.base_ref != '' }}
|
|
||||||
run: |
|
|
||||||
# Add merge target branch to determine diff.
|
|
||||||
# This is not needed for build triggered by tags, since there will be no code diff.
|
|
||||||
git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
|
|
||||||
|
|
||||||
- name: Setup diff-test-coverage
|
|
||||||
run: npm install @connectis/diff-test-coverage@1.5.3
|
|
||||||
|
|
||||||
- name: "(openjdk8) indexing modules test"
|
|
||||||
env:
|
|
||||||
MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
|
|
||||||
run: ./.github/scripts/unit_tests_script.sh
|
|
||||||
|
|
||||||
- name: "(openjdk8) processing module test"
|
|
||||||
env:
|
|
||||||
MAVEN_PROJECTS: processing
|
|
||||||
run: ./.github/scripts/unit_tests_script.sh
|
|
||||||
|
|
||||||
- name: "(openjdk8) server module test"
|
|
||||||
env:
|
|
||||||
MAVEN_PROJECTS: server
|
|
||||||
run: ./.github/scripts/unit_tests_script.sh
|
|
||||||
|
|
||||||
- name: "(openjdk8) other modules test"
|
|
||||||
env:
|
|
||||||
MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
|
|
||||||
run: ./.github/scripts/unit_tests_script.sh
|
|
||||||
|
|
|
@ -78,7 +78,7 @@ services:
|
||||||
druid-it-net:
|
druid-it-net:
|
||||||
ipv4_address: 172.172.172.14
|
ipv4_address: 172.172.172.14
|
||||||
ports:
|
ports:
|
||||||
- 8084:8083
|
- 8086:8083 #8084 is used by mono service on GHA runners
|
||||||
- 8284:8283
|
- 8284:8283
|
||||||
- 5010:5007
|
- 5010:5007
|
||||||
privileged: true
|
privileged: true
|
||||||
|
|
|
@ -78,7 +78,7 @@ services:
|
||||||
druid-it-net:
|
druid-it-net:
|
||||||
ipv4_address: 172.172.172.14
|
ipv4_address: 172.172.172.14
|
||||||
ports:
|
ports:
|
||||||
- 8084:8083
|
- 8086:8083 #8084 is used by mono service on GHA runners
|
||||||
- 8284:8283
|
- 8284:8283
|
||||||
- 5010:5007
|
- 5010:5007
|
||||||
privileged: true
|
privileged: true
|
||||||
|
|
|
@ -108,6 +108,8 @@ public class ITJdbcQueryTest
|
||||||
};
|
};
|
||||||
// ensure that wikipedia segments are loaded completely
|
// ensure that wikipedia segments are loaded completely
|
||||||
dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
|
dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
|
||||||
|
dataLoaderHelper.waitUntilDatasourceIsReady("wikipedia");
|
||||||
|
dataLoaderHelper.waitUntilDatasourceIsReady("twitterstream");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
12
it.sh
12
it.sh
|
@ -31,6 +31,8 @@ function usage
|
||||||
{
|
{
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
Usage: $0 cmd [category]
|
Usage: $0 cmd [category]
|
||||||
|
ci
|
||||||
|
build Druid and the distribution for CI pipelines
|
||||||
build
|
build
|
||||||
build Druid and the distribution
|
build Druid and the distribution
|
||||||
dist
|
dist
|
||||||
|
@ -49,6 +51,8 @@ Usage: $0 cmd [category]
|
||||||
show the last 20 lines of each container log
|
show the last 20 lines of each container log
|
||||||
travis <category>
|
travis <category>
|
||||||
run one IT in Travis (build dist, image, run test, tail logs)
|
run one IT in Travis (build dist, image, run test, tail logs)
|
||||||
|
github <category>
|
||||||
|
run one IT in Github Workflows (run test, tail logs)
|
||||||
prune
|
prune
|
||||||
prune Docker volumes
|
prune Docker volumes
|
||||||
|
|
||||||
|
@ -189,6 +193,9 @@ case $CMD in
|
||||||
"help" )
|
"help" )
|
||||||
usage
|
usage
|
||||||
;;
|
;;
|
||||||
|
"ci" )
|
||||||
|
mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE -T1.0C
|
||||||
|
;;
|
||||||
"build" )
|
"build" )
|
||||||
mvn clean package -P dist $MAVEN_IGNORE -T1.0C
|
mvn clean package -P dist $MAVEN_IGNORE -T1.0C
|
||||||
;;
|
;;
|
||||||
|
@ -230,6 +237,11 @@ case $CMD in
|
||||||
$0 test $CATEGORY
|
$0 test $CATEGORY
|
||||||
$0 tail $CATEGORY
|
$0 tail $CATEGORY
|
||||||
;;
|
;;
|
||||||
|
"github" )
|
||||||
|
prepare_category $1
|
||||||
|
$0 test $CATEGORY
|
||||||
|
$0 tail $CATEGORY
|
||||||
|
;;
|
||||||
"prune" )
|
"prune" )
|
||||||
# Caution: this removes all volumes, which is generally what you
|
# Caution: this removes all volumes, which is generally what you
|
||||||
# want when testing.
|
# want when testing.
|
||||||
|
|
Loading…
Reference in New Issue