Enforce code coverage (#9863)

* Enforce code coverage Add an automated way of checking if new code has adequate unit tests, since merging code coverage reports and check coverage thresholds via coveralls or codecov is unreliable. The following minimum unit test code coverage is now enforced: - 80% functions - 65% branch - 65% line Branch and line coverage thresholds are slightly lower for now as they are harder to achieve. After the code coverage check looks reliable, the thresholds can be increased later if needed. * Add comments
2020-05-20 09:31:37 -07:00 · 2020-05-20 09:31:37 -07:00 · 427239f451
parent 5b4b5d77a8
commit 427239f451
3 changed files with 65 additions and 47 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -81,8 +81,7 @@ jobs:
        - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
        - distribution/bin/check-licenses.py licenses.yaml target/license-reports

-    - &compile_strict
-      name: "(openjdk8) strict compilation"
+    - name: "(openjdk8) strict compilation"
      install: skip
      # Strict compilation requires more than 2 GB
      script: >
@ -136,20 +135,52 @@ jobs:

    - &test_processing_module
      name: "(openjdk8) processing module test"
-      env: &processing_env
+      env:
      - MAVEN_PROJECTS='processing'
-      before_script: &setup_java_test
+      before_script:
+        - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=true
+      script:
        - unset _JAVA_OPTIONS
-      script: &run_java_test
        # Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when
        # resolving the SIGAR dependency.
        - >
          MAVEN_OPTS='-Xmx800m' ${MVN} test -pl ${MAVEN_PROJECTS}
-          ${MAVEN_SKIP} -Dremoteresources.skip=true
+          ${MAVEN_SKIP} -Dremoteresources.skip=true -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
        - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
        - free -m
-      after_success: &upload_java_unit_test_coverage
        - ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report
+        # Add merge target branch to determine diff (see https://github.com/travis-ci/travis-ci/issues/6069)
+        - echo "TRAVIS_BRANCH=${TRAVIS_BRANCH}"  # for debugging
+        - git remote set-branches --add origin ${TRAVIS_BRANCH} && git fetch
+        # Determine the modified files that match the maven projects being tested. We use maven project lists that
+        # either exclude (starts with "!") or include (does not start with "!"), so both cases need to be handled.
+        - all_files="$(git diff --name-only origin/${TRAVIS_BRANCH}...HEAD | grep "\.java$" || [[ $? == 1 ]])"
+        - for f in ${all_files}; do echo $f; done  # for debugging
+        - >
+          if [[ "${MAVEN_PROJECTS}" = \!* ]]; then
+          regex="${MAVEN_PROJECTS:1}";
+          regex="^${regex//,\!/\\|^}";
+          project_files="$(echo "${all_files}" | grep -v "${regex}" || [[ $? == 1 ]])";
+          else
+          regex="^${MAVEN_PROJECTS//,/\\|^}";
+          project_files="$(echo "${all_files}" | grep "${regex}" || [[ $? == 1 ]])";
+          fi
+        - for f in ${project_files}; do echo $f; done  # for debugging
+        # Check diff code coverage for the maven projects being tested (retry install in case of network error)
+        - >
+          if [ -n "${project_files}" ]; then
+          travis_retry npm install @connectis/diff-test-coverage@1.5.3
+          && git diff origin/${TRAVIS_BRANCH}...HEAD -- ${project_files}
+          | node_modules/.bin/diff-test-coverage
+          --coverage "**/target/site/jacoco/jacoco.xml"
+          --type jacoco
+          --line-coverage 65
+          --branch-coverage 65
+          --function-coverage 80
+          --
+          || { printf "\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\n" && false; }
+          fi
+      after_success:
        # retry in case of network error
        - travis_retry curl -o codecov.sh -s https://codecov.io/bash
        - travis_retry bash codecov.sh -X gcov
@ -159,89 +190,69 @@ jobs:
      jdk: openjdk11

    - &test_processing_module_sqlcompat
+      <<: *test_processing_module
      name: "(openjdk8) processing module test (SQL Compatibility)"
-      env: *processing_env
-      before_script: *setup_java_test
-      script: &run_java_sql_compat_test
-        # Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when
-        # resolving the SIGAR dependency.
-        - >
-          MAVEN_OPTS='-Xmx800m' ${MVN} test -pl ${MAVEN_PROJECTS} -Ddruid.generic.useDefaultValueForNull=false
-          ${MAVEN_SKIP} -Dremoteresources.skip=true
-        - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
-        - free -m
-      after_success: *upload_java_unit_test_coverage
+      before_script: &setup_sqlcompat
+      - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false

    - <<: *test_processing_module_sqlcompat
      name: "(openjdk11) processing module test (SQL Compatibility)"
      jdk: openjdk11

    - &test_indexing_module
+      <<: *test_processing_module
      name: "(openjdk8) indexing modules test"
-      env: &indexing_env
+      env:
      - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
-      before_script: *setup_java_test
-      script: *run_java_test
-      after_success: *upload_java_unit_test_coverage

    - <<: *test_indexing_module
      name: "(openjdk11) indexing modules test"
      jdk: openjdk11

    - &test_indexing_module_sqlcompat
+      <<: *test_indexing_module
      name: "(openjdk8) indexing modules test (SQL Compatibility)"
-      env: *indexing_env
-      before_script: *setup_java_test
-      script: *run_java_sql_compat_test
-      after_success: *upload_java_unit_test_coverage
+      before_script: *setup_sqlcompat

    - <<: *test_indexing_module_sqlcompat
      name: "(openjdk11) indexing modules test (SQL Compatibility)"
      jdk: openjdk11

    - &test_server_module
+      <<: *test_processing_module
      name: "(openjdk8) server module test"
-      env: &server_env
+      env:
        - MAVEN_PROJECTS='server'
-      before_script: *setup_java_test
-      script: *run_java_test
-      after_success: *upload_java_unit_test_coverage

    - <<: *test_server_module
      name: "(openjdk11) server module test"
      jdk: openjdk11

    - &test_server_module_sqlcompat
+      <<: *test_server_module
      name: "(openjdk8) server module test (SQL Compatibility)"
-      env: *server_env
-      before_script: *setup_java_test
-      script: *run_java_sql_compat_test
-      after_success: *upload_java_unit_test_coverage
+      before_script: *setup_sqlcompat

    - <<: *test_server_module_sqlcompat
      name: "(openjdk11) server module test (SQL Compatibility)"
      jdk: openjdk11

-    - &test_modules
+    - &test_other_modules
+      <<: *test_processing_module
      name: "(openjdk8) other modules test"
-      env: &other_env
+      env:
        - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console'
-      before_script: *setup_java_test
-      script: *run_java_test
-      after_success: *upload_java_unit_test_coverage

-    - <<: *test_modules
+    - <<: *test_other_modules
      name: "(openjdk11) other modules test"
      jdk: openjdk11

-    - &test_modules_sqlcompat
+    - &test_other_modules_sqlcompat
+      <<: *test_other_modules
      name: "(openjdk8) other modules test (SQL Compatibility)"
-      env: *other_env
-      before_script: *setup_java_test
-      script: *run_java_sql_compat_test
-      after_success: *upload_java_unit_test_coverage
+      before_script: *setup_sqlcompat

-    - <<: *test_modules_sqlcompat
+    - <<: *test_other_modules_sqlcompat
      name: "(openjdk11) other modules test (SQL Compatibility)"
      jdk: openjdk11

--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@ -220,6 +220,13 @@
          <skip>true</skip>
        </configuration>
      </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <configuration>
+          <skip>true</skip> <!-- ignore non-production code -->
+        </configuration>
+      </plugin>
    </plugins>
    <pluginManagement>
      <plugins>
--- a/pom.xml
+++ b/pom.xml
@ -1251,7 +1251,7 @@
            <plugin>
                <groupId>org.jacoco</groupId>
                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
+                <version>0.8.5</version>
                <configuration>
                    <excludes>
                        <!-- Ignore generated code -->