Migrate jdk8 unit tests from Travis to GHA (#13518)

* migrate UTs form Travis to GHA

* update permissions

* rename file

* set fetch depth to 1

* debugs remote branches

* test with github.ref variable

* fetch github.base_ref for diff

* nit

* test git diff

* run tests

* test code coverage failure scenario

* nit

* nit

* revert code changes

* revert code changes

* Setup diff-test-coverage before tests

* build distribution module at end in packaging check

* nit

* remove redundant steps in static-checks workflow

* drop jdk8 unit tests from Travis

.github/scripts/unit_tests_script.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+unset _JAVA_OPTIONS
+
+# Set MAVEN_OPTS for Surefire launcher.
+MAVEN_OPTS='-Xmx2048m' ${MVN} test -pl ${MAVEN_PROJECTS} \
+${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
+sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
+free -m
+${MVN} -pl ${MAVEN_PROJECTS} jacoco:report
+
+# Determine the modified files that match the maven projects being tested. We use maven project lists that
+# either exclude (starts with "!") or include (does not start with "!"), so both cases need to be handled.
+# If the build is triggered by a tag, an error will be printed, but `all_files` will be correctly set to empty
+# so that the coverage check is skipped.
+all_files="$(git diff --name-only origin/${GITHUB_BASE_REF}...HEAD | grep "\.java$" || [[ $? == 1 ]])"
+
+echo "Changed files:"
+for f in ${all_files}
+do
+  echo $f # for debugging
+done
+
+if [[ "${MAVEN_PROJECTS}" = \!* ]]
+then
+  regex="${MAVEN_PROJECTS:1}"
+  regex="^${regex//,\!/\\|^}"
+  project_files="$(echo "${all_files}" | grep -v "${regex}" || [[ $? == 1 ]])"
+else
+  regex="^${MAVEN_PROJECTS//,/\\|^}"
+  project_files="$(echo "${all_files}" | grep "${regex}" || [[ $? == 1 ]])"
+fi
+
+echo "Changed files in current project:"
+for f in ${project_files}
+do
+  echo $f # for debugging
+done
+
+# Check diff code coverage for the maven projects being tested (retry install in case of network error).
+# Currently, the function coverage check is not reliable, so it is disabled.
+if [ -n "${project_files}" ]
+then
+  git diff origin/${GITHUB_BASE_REF}...HEAD -- ${project_files} |
+  node_modules/.bin/diff-test-coverage \
+  --coverage "**/target/site/jacoco/jacoco.xml" \
+  --type jacoco \
+  --line-coverage 50 \
+  --branch-coverage 50 \
+  --function-coverage 0 \
+  --log-template "coverage-lines-complete" \
+  --log-template "coverage-files-complete" \
+  --log-template "totals-complete" \
+  --log-template "errors" \
+  -- ||
+  { printf "\n\n****FAILED****\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && false; }
+fi
+
+{ for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done }
+{ for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done }

.github/workflows/static-checks.yml

@@ -59,8 +59,10 @@ jobs:
       - name: packaging check
         run: |
           ./.github/scripts/setup_generate_license.sh
-          ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
+          ${MVN} clean install -Prat -Pbundle-contrib-exts --fail-at-end \
-          -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
+          -pl '!benchmarks, !distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
+          ${MVN} install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
+          -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
 
       - name: script checks
         # who watches the watchers?
@@ -163,10 +165,6 @@ jobs:
         if: ${{ matrix.java == 'jdk8' }}
         run: |
           ./.github/scripts/setup_generate_license.sh
-          sudo apt-get update && sudo apt-get install python3 -y
-          curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
-          pip3 install wheel  # install wheel first explicitly
-          pip3 install --upgrade pyyaml
           web-console/script/druid build
           web-console/script/druid start
           (cd web-console && npm run test-e2e)

.github/workflows/unit-tests.yml

@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: (openjdk8) Unit Tests CI
+on:
+  push:
+    branches:
+      - master
+      - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+  pull_request:
+    branches:
+      - master
+      - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+env:
+  MVN: mvn -B
+  MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
+  MAVEN_SKIP_TESTS: -P skip-tests
+  MAVEN_OPTS: -Xmx3000m
+  FORCE_COLOR: 2
+
+jobs:
+  unit-tests:
+    strategy:
+      matrix:
+        DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout branch
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: setup jdk8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: 8
+          cache: 'maven'
+
+      - name: maven install
+        run: |
+          echo 'Running Maven install...' &&
+          ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
+          ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+
+      - name: setup variables
+        run: |
+          export base_ref=${{ github.base_ref }}
+          echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
+          export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
+          echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
+
+      - name: fetch base branch for test coverage
+        if: ${{ github.base_ref != '' }}
+        run: |
+          # Add merge target branch to determine diff.
+          # This is not needed for build triggered by tags, since there will be no code diff.
+          git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
+
+      - name: Setup diff-test-coverage
+        run: npm install @connectis/diff-test-coverage@1.5.3
+
+      - name: "(openjdk8) indexing modules test"
+        env:
+          MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
+        run: ./.github/scripts/unit_tests_script.sh
+
+      - name: "(openjdk8) processing module test"
+        env:
+          MAVEN_PROJECTS: processing
+        run: ./.github/scripts/unit_tests_script.sh
+
+      - name: "(openjdk8) server module test"
+        env:
+          MAVEN_PROJECTS: server
+        run: ./.github/scripts/unit_tests_script.sh
+
+      - name: "(openjdk8) other modules test"
+        env:
+          MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
+        run: ./.github/scripts/unit_tests_script.sh

.travis.yml

@@ -49,7 +49,7 @@ addons:
 install: ./check_test_suite.py && travis_terminate 0  || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
 # There are 3 stages of tests
-# 1. Tests - phase 1
+# 1. Tests - phase 1 (completely migrated to github workflows)
 # 2. Tests - phase 2
 # 3. cron
 #
@@ -71,8 +71,9 @@ stages:
 jobs:
   include:
     - &test_processing_module
-      name: "(openjdk8) processing module test"
+      name: "(openjdk11) processing module test"
-      stage: Tests - phase 1
+      stage: Tests - phase 2
+      jdk: openjdk11
       env:
       - MAVEN_PROJECTS='processing'
       before_script:
@@ -131,11 +132,6 @@ jobs:
         - travis_retry curl -o codecov.sh -s https://codecov.io/bash
         - travis_retry bash codecov.sh -X gcov
 
-    - <<: *test_processing_module
-      name: "(openjdk11) processing module test"
-      stage: Tests - phase 2
-      jdk: openjdk11
-
     - <<: *test_processing_module
       name: "(openjdk17) processing module test"
       stage: Tests - phase 2
@@ -143,15 +139,11 @@ jobs:
 
     - &test_processing_module_sqlcompat
       <<: *test_processing_module
-      name: "(openjdk8) processing module test (SQL Compatibility)"
-      stage: Tests - phase 1
-      before_script: &setup_sqlcompat
-        - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false
-
-    - <<: *test_processing_module_sqlcompat
       name: "(openjdk11) processing module test (SQL Compatibility)"
       stage: Tests - phase 2
       jdk: openjdk11
+      before_script: &setup_sqlcompat
+        - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false
 
     - <<: *test_processing_module_sqlcompat
       name: "(openjdk17) processing module test (SQL Compatibility)"
@@ -160,14 +152,11 @@ jobs:
 
     - &test_indexing_module
       <<: *test_processing_module
-      name: "(openjdk8) indexing modules test"
-      env:
-        - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
-
-    - <<: *test_indexing_module
       name: "(openjdk11) indexing modules test"
       stage: Tests - phase 2
       jdk: openjdk11
+      env:
+        - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
 
     - <<: *test_indexing_module
       name: "(openjdk17) indexing modules test"
@@ -176,14 +165,10 @@ jobs:
 
     - &test_indexing_module_sqlcompat
       <<: *test_indexing_module
-      name: "(openjdk8) indexing modules test (SQL Compatibility)"
-      stage: Tests - phase 1
-      before_script: *setup_sqlcompat
-
-    - <<: *test_indexing_module_sqlcompat
       name: "(openjdk11) indexing modules test (SQL Compatibility)"
       stage: Tests - phase 2
       jdk: openjdk11
+      before_script: *setup_sqlcompat
 
     - <<: *test_indexing_module_sqlcompat
       name: "(openjdk17) indexing modules test (SQL Compatibility)"
@@ -192,14 +177,11 @@ jobs:
 
     - &test_server_module
       <<: *test_processing_module
-      name: "(openjdk8) server module test"
-      env:
-        - MAVEN_PROJECTS='server'
-
-    - <<: *test_server_module
       name: "(openjdk11) server module test"
       stage: Tests - phase 2
       jdk: openjdk11
+      env:
+        - MAVEN_PROJECTS='server'
 
     - <<: *test_server_module
       name: "(openjdk17) server module test"
@@ -208,13 +190,10 @@ jobs:
 
     - &test_server_module_sqlcompat
       <<: *test_server_module
-      name: "(openjdk8) server module test (SQL Compatibility)"
-      before_script: *setup_sqlcompat
-
-    - <<: *test_server_module_sqlcompat
       name: "(openjdk11) server module test (SQL Compatibility)"
       stage: Tests - phase 2
       jdk: openjdk11
+      before_script: *setup_sqlcompat
 
     - <<: *test_server_module_sqlcompat
       name: "(openjdk17) server module test (SQL Compatibility)"
@@ -223,14 +202,11 @@ jobs:
 
     - &test_other_modules
       <<: *test_processing_module
-      name: "(openjdk8) other modules test"
-      env:
-        - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
-
-    - <<: *test_other_modules
       name: "(openjdk11) other modules test"
       stage: Tests - phase 2
       jdk: openjdk11
+      env:
+        - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
 
     - <<: *test_other_modules
       name: "(openjdk17) other modules test"
@@ -239,13 +215,10 @@ jobs:
 
     - &test_other_modules_sqlcompat
       <<: *test_other_modules
-      name: "(openjdk8) other modules test (SQL Compatibility)"
-      before_script: *setup_sqlcompat
-
-    - <<: *test_other_modules_sqlcompat
       name: "(openjdk11) other modules test (SQL Compatibility)"
       stage: Tests - phase 2
       jdk: openjdk11
+      before_script: *setup_sqlcompat
 
     - <<: *test_other_modules_sqlcompat
       name: "(openjdk17) other modules test (SQL Compatibility)"