From 4368b3a071719b50e8341626d967b8ae77caa786 Mon Sep 17 00:00:00 2001 From: Tejaswini Bandlamudi <96047043+tejaswini-imply@users.noreply.github.com> Date: Fri, 13 Jan 2023 14:46:58 +0530 Subject: [PATCH] Migrate jdk8 unit tests from Travis to GHA (#13518) * migrate UTs form Travis to GHA * update permissions * rename file * set fetch depth to 1 * debugs remote branches * test with github.ref variable * fetch github.base_ref for diff * nit * test git diff * run tests * test code coverage failure scenario * nit * nit * revert code changes * revert code changes * Setup diff-test-coverage before tests * build distribution module at end in packaging check * nit * remove redundant steps in static-checks workflow * drop jdk8 unit tests from Travis --- .github/scripts/unit_tests_script.sh | 77 ++++++++++++++++++++++ .github/workflows/static-checks.yml | 10 ++- .github/workflows/unit-tests.yml | 98 ++++++++++++++++++++++++++++ .travis.yml | 57 +++++----------- 4 files changed, 194 insertions(+), 48 deletions(-) create mode 100755 .github/scripts/unit_tests_script.sh create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/scripts/unit_tests_script.sh b/.github/scripts/unit_tests_script.sh new file mode 100755 index 00000000000..f27bc4bad9c --- /dev/null +++ b/.github/scripts/unit_tests_script.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +unset _JAVA_OPTIONS + +# Set MAVEN_OPTS for Surefire launcher. +MAVEN_OPTS='-Xmx2048m' ${MVN} test -pl ${MAVEN_PROJECTS} \ +${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} +sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0" +free -m +${MVN} -pl ${MAVEN_PROJECTS} jacoco:report + +# Determine the modified files that match the maven projects being tested. We use maven project lists that +# either exclude (starts with "!") or include (does not start with "!"), so both cases need to be handled. +# If the build is triggered by a tag, an error will be printed, but `all_files` will be correctly set to empty +# so that the coverage check is skipped. +all_files="$(git diff --name-only origin/${GITHUB_BASE_REF}...HEAD | grep "\.java$" || [[ $? == 1 ]])" + +echo "Changed files:" +for f in ${all_files} +do + echo $f # for debugging +done + +if [[ "${MAVEN_PROJECTS}" = \!* ]] +then + regex="${MAVEN_PROJECTS:1}" + regex="^${regex//,\!/\\|^}" + project_files="$(echo "${all_files}" | grep -v "${regex}" || [[ $? == 1 ]])" +else + regex="^${MAVEN_PROJECTS//,/\\|^}" + project_files="$(echo "${all_files}" | grep "${regex}" || [[ $? == 1 ]])" +fi + +echo "Changed files in current project:" +for f in ${project_files} +do + echo $f # for debugging +done + +# Check diff code coverage for the maven projects being tested (retry install in case of network error). +# Currently, the function coverage check is not reliable, so it is disabled. +if [ -n "${project_files}" ] +then + git diff origin/${GITHUB_BASE_REF}...HEAD -- ${project_files} | + node_modules/.bin/diff-test-coverage \ + --coverage "**/target/site/jacoco/jacoco.xml" \ + --type jacoco \ + --line-coverage 50 \ + --branch-coverage 50 \ + --function-coverage 0 \ + --log-template "coverage-lines-complete" \ + --log-template "coverage-files-complete" \ + --log-template "totals-complete" \ + --log-template "errors" \ + -- || + { printf "\n\n****FAILED****\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && false; } +fi + +{ for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done } +{ for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done } diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 37e7a222585..f247bf01a19 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -59,8 +59,10 @@ jobs: - name: packaging check run: | ./.github/scripts/setup_generate_license.sh - ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \ - -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C + ${MVN} clean install -Prat -Pbundle-contrib-exts --fail-at-end \ + -pl '!benchmarks, !distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C + ${MVN} install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \ + -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C - name: script checks # who watches the watchers? @@ -163,10 +165,6 @@ jobs: if: ${{ matrix.java == 'jdk8' }} run: | ./.github/scripts/setup_generate_license.sh - sudo apt-get update && sudo apt-get install python3 -y - curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 - pip3 install wheel # install wheel first explicitly - pip3 install --upgrade pyyaml web-console/script/druid build web-console/script/druid start (cd web-console && npm run test-e2e) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000000..9635fb44b77 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: (openjdk8) Unit Tests CI +on: + push: + branches: + - master + - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches + pull_request: + branches: + - master + - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + MVN: mvn -B + MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true + MAVEN_SKIP_TESTS: -P skip-tests + MAVEN_OPTS: -Xmx3000m + FORCE_COLOR: 2 + +jobs: + unit-tests: + strategy: + matrix: + DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ] + runs-on: ubuntu-latest + steps: + - name: checkout branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: setup jdk8 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 8 + cache: 'maven' + + - name: maven install + run: | + echo 'Running Maven install...' && + ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && + ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + + - name: setup variables + run: | + export base_ref=${{ github.base_ref }} + echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV + export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }} + echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV + + - name: fetch base branch for test coverage + if: ${{ github.base_ref != '' }} + run: | + # Add merge target branch to determine diff. + # This is not needed for build triggered by tags, since there will be no code diff. + git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch + + - name: Setup diff-test-coverage + run: npm install @connectis/diff-test-coverage@1.5.3 + + - name: "(openjdk8) indexing modules test" + env: + MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service + run: ./.github/scripts/unit_tests_script.sh + + - name: "(openjdk8) processing module test" + env: + MAVEN_PROJECTS: processing + run: ./.github/scripts/unit_tests_script.sh + + - name: "(openjdk8) server module test" + env: + MAVEN_PROJECTS: server + run: ./.github/scripts/unit_tests_script.sh + + - name: "(openjdk8) other modules test" + env: + MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' + run: ./.github/scripts/unit_tests_script.sh diff --git a/.travis.yml b/.travis.yml index 003331be2e3..c9afd77e14b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ addons: install: ./check_test_suite.py && travis_terminate 0 || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} # There are 3 stages of tests -# 1. Tests - phase 1 +# 1. Tests - phase 1 (completely migrated to github workflows) # 2. Tests - phase 2 # 3. cron # @@ -71,8 +71,9 @@ stages: jobs: include: - &test_processing_module - name: "(openjdk8) processing module test" - stage: Tests - phase 1 + name: "(openjdk11) processing module test" + stage: Tests - phase 2 + jdk: openjdk11 env: - MAVEN_PROJECTS='processing' before_script: @@ -131,11 +132,6 @@ jobs: - travis_retry curl -o codecov.sh -s https://codecov.io/bash - travis_retry bash codecov.sh -X gcov - - <<: *test_processing_module - name: "(openjdk11) processing module test" - stage: Tests - phase 2 - jdk: openjdk11 - - <<: *test_processing_module name: "(openjdk17) processing module test" stage: Tests - phase 2 @@ -143,15 +139,11 @@ jobs: - &test_processing_module_sqlcompat <<: *test_processing_module - name: "(openjdk8) processing module test (SQL Compatibility)" - stage: Tests - phase 1 - before_script: &setup_sqlcompat - - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false - - - <<: *test_processing_module_sqlcompat name: "(openjdk11) processing module test (SQL Compatibility)" stage: Tests - phase 2 jdk: openjdk11 + before_script: &setup_sqlcompat + - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false - <<: *test_processing_module_sqlcompat name: "(openjdk17) processing module test (SQL Compatibility)" @@ -160,14 +152,11 @@ jobs: - &test_indexing_module <<: *test_processing_module - name: "(openjdk8) indexing modules test" - env: - - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service' - - - <<: *test_indexing_module name: "(openjdk11) indexing modules test" stage: Tests - phase 2 jdk: openjdk11 + env: + - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service' - <<: *test_indexing_module name: "(openjdk17) indexing modules test" @@ -176,14 +165,10 @@ jobs: - &test_indexing_module_sqlcompat <<: *test_indexing_module - name: "(openjdk8) indexing modules test (SQL Compatibility)" - stage: Tests - phase 1 - before_script: *setup_sqlcompat - - - <<: *test_indexing_module_sqlcompat name: "(openjdk11) indexing modules test (SQL Compatibility)" stage: Tests - phase 2 jdk: openjdk11 + before_script: *setup_sqlcompat - <<: *test_indexing_module_sqlcompat name: "(openjdk17) indexing modules test (SQL Compatibility)" @@ -192,14 +177,11 @@ jobs: - &test_server_module <<: *test_processing_module - name: "(openjdk8) server module test" - env: - - MAVEN_PROJECTS='server' - - - <<: *test_server_module name: "(openjdk11) server module test" stage: Tests - phase 2 jdk: openjdk11 + env: + - MAVEN_PROJECTS='server' - <<: *test_server_module name: "(openjdk17) server module test" @@ -208,13 +190,10 @@ jobs: - &test_server_module_sqlcompat <<: *test_server_module - name: "(openjdk8) server module test (SQL Compatibility)" - before_script: *setup_sqlcompat - - - <<: *test_server_module_sqlcompat name: "(openjdk11) server module test (SQL Compatibility)" stage: Tests - phase 2 jdk: openjdk11 + before_script: *setup_sqlcompat - <<: *test_server_module_sqlcompat name: "(openjdk17) server module test (SQL Compatibility)" @@ -223,14 +202,11 @@ jobs: - &test_other_modules <<: *test_processing_module - name: "(openjdk8) other modules test" - env: - - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' - - - <<: *test_other_modules name: "(openjdk11) other modules test" stage: Tests - phase 2 jdk: openjdk11 + env: + - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' - <<: *test_other_modules name: "(openjdk17) other modules test" @@ -239,13 +215,10 @@ jobs: - &test_other_modules_sqlcompat <<: *test_other_modules - name: "(openjdk8) other modules test (SQL Compatibility)" - before_script: *setup_sqlcompat - - - <<: *test_other_modules_sqlcompat name: "(openjdk11) other modules test (SQL Compatibility)" stage: Tests - phase 2 jdk: openjdk11 + before_script: *setup_sqlcompat - <<: *test_other_modules_sqlcompat name: "(openjdk17) other modules test (SQL Compatibility)"