mirror of https://github.com/apache/druid.git
414 lines
17 KiB
YAML
414 lines
17 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
language: java
|
|
|
|
sudo: true
|
|
dist: xenial
|
|
|
|
jdk:
|
|
- openjdk8
|
|
|
|
cache:
|
|
directories:
|
|
- $HOME/.m2
|
|
|
|
env:
|
|
global:
|
|
- DOCKER_IP=127.0.0.1 # for integration tests
|
|
- MVN="mvn -B"
|
|
- > # Various options to make execution of maven goals faster (e.g., mvn install)
|
|
MAVEN_SKIP="
|
|
-Danimal.sniffer.skip=true
|
|
-Dcheckstyle.skip=true
|
|
-Ddruid.console.skip=true
|
|
-Denforcer.skip=true
|
|
-Dforbiddenapis.skip=true
|
|
-Dmaven.javadoc.skip=true
|
|
-Dpmd.skip=true
|
|
-Dspotbugs.skip=true
|
|
"
|
|
- MAVEN_SKIP_TESTS="-DskipTests -Djacoco.skip=true"
|
|
|
|
# Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
|
|
# needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
|
|
# output. To compensate, use travis_wait to extend the timeout.
|
|
install: MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C
|
|
|
|
stages:
|
|
- name: test # jobs that do not specify a stage get this default value
|
|
if: type != cron
|
|
- name: cron
|
|
if: type = cron
|
|
|
|
jobs:
|
|
include:
|
|
- name: "animal sniffer checks"
|
|
script: ${MVN} animal-sniffer:check --fail-at-end
|
|
|
|
- name: "checkstyle"
|
|
script: ${MVN} checkstyle:checkstyle --fail-at-end
|
|
|
|
- name: "enforcer checks"
|
|
script: ${MVN} enforcer:enforce --fail-at-end
|
|
|
|
- name: "forbidden api checks"
|
|
script: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
|
|
|
|
- name: "pmd checks"
|
|
script: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check
|
|
|
|
- name: "spotbugs checks"
|
|
script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
|
|
|
|
- name: "license checks"
|
|
install: skip
|
|
before_script: &setup_generate_license
|
|
- sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y
|
|
- pip3 install wheel # install wheel first explicitly
|
|
- pip3 install pyyaml
|
|
script:
|
|
- >
|
|
${MVN} apache-rat:check -Prat --fail-at-end
|
|
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
|
|
-Drat.consoleOutput=true
|
|
# Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available
|
|
# (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system).
|
|
- mkdir -p target
|
|
- distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
|
|
- distribution/bin/check-licenses.py licenses.yaml target/license-reports
|
|
|
|
- &compile_strict
|
|
name: "(openjdk8) strict compilation"
|
|
install: skip
|
|
# Strict compilation requires more than 2 GB
|
|
script: >
|
|
MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end
|
|
-pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
|
|
|
|
- name: "analyze dependencies"
|
|
script: |-
|
|
MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo "
|
|
|
|
The dependency analysis has found a dependency that is either:
|
|
|
|
1) Used and undeclared: These are available as a transitive dependency but should be explicitly
|
|
added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is
|
|
shown above.
|
|
|
|
2) Unused and declared: These are not needed and removing them from the POM will speed up the build
|
|
and reduce the artifact size. The dependencies to remove are shown above.
|
|
|
|
If there are false positive dependency analysis warnings, they can be suppressed:
|
|
https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
|
|
https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
|
|
|
|
For more information, refer to:
|
|
https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
|
|
|
|
" && false; }
|
|
|
|
- name: "intellij inspections"
|
|
script: >
|
|
docker run --rm
|
|
-v $(pwd):/project
|
|
-v ~/.m2:/home/inspect/.m2
|
|
ccaominh/intellij-inspect:1.0.0
|
|
/project/pom.xml
|
|
/project/.idea/inspectionProfiles/Druid.xml
|
|
--levels ERROR
|
|
--scope JavaInspectionsScope
|
|
|
|
- &package
|
|
name: "(openjdk8) packaging check"
|
|
install: skip
|
|
before_script: *setup_generate_license
|
|
script: >
|
|
MAVEN_OPTS='-Xmx3000m' ${MVN} clean install -Pdist -Pbundle-contrib-exts --fail-at-end
|
|
-pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Ddruid.console.skip=false -T1C
|
|
|
|
- <<: *package
|
|
name: "(openjdk11) packaging check"
|
|
jdk: openjdk11
|
|
|
|
- &test_processing_module
|
|
name: "(openjdk8) processing module test"
|
|
env: &processing_env
|
|
- MAVEN_PROJECTS='processing'
|
|
before_script: &setup_java_test
|
|
- unset _JAVA_OPTIONS
|
|
script: &run_java_test
|
|
# Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when
|
|
# resolving the SIGAR dependency.
|
|
- >
|
|
MAVEN_OPTS='-Xmx800m' ${MVN} test -pl ${MAVEN_PROJECTS}
|
|
${MAVEN_SKIP} -Dremoteresources.skip=true
|
|
- sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
|
|
- free -m
|
|
after_success: &upload_java_unit_test_coverage
|
|
- ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report
|
|
# retry in case of network error
|
|
- travis_retry curl -o codecov.sh -s https://codecov.io/bash
|
|
- travis_retry bash codecov.sh -X gcov
|
|
|
|
- <<: *test_processing_module
|
|
name: "(openjdk11) processing module test"
|
|
jdk: openjdk11
|
|
|
|
- &test_processing_module_sqlcompat
|
|
name: "(openjdk8) processing module test (SQL Compatibility)"
|
|
env: *processing_env
|
|
before_script: *setup_java_test
|
|
script: &run_java_sql_compat_test
|
|
# Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when
|
|
# resolving the SIGAR dependency.
|
|
- >
|
|
MAVEN_OPTS='-Xmx800m' ${MVN} test -pl ${MAVEN_PROJECTS} -Ddruid.generic.useDefaultValueForNull=false
|
|
${MAVEN_SKIP} -Dremoteresources.skip=true
|
|
- sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
|
|
- free -m
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_processing_module_sqlcompat
|
|
name: "(openjdk11) processing module test (SQL Compatibility)"
|
|
jdk: openjdk11
|
|
|
|
- &test_indexing_module
|
|
name: "(openjdk8) indexing modules test"
|
|
env: &indexing_env
|
|
- MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
|
|
before_script: *setup_java_test
|
|
script: *run_java_test
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_indexing_module
|
|
name: "(openjdk11) indexing modules test"
|
|
jdk: openjdk11
|
|
|
|
- &test_indexing_module_sqlcompat
|
|
name: "(openjdk8) indexing modules test (SQL Compatibility)"
|
|
env: *indexing_env
|
|
before_script: *setup_java_test
|
|
script: *run_java_sql_compat_test
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_indexing_module_sqlcompat
|
|
name: "(openjdk11) indexing modules test (SQL Compatibility)"
|
|
jdk: openjdk11
|
|
|
|
- &test_server_module
|
|
name: "(openjdk8) server module test"
|
|
env: &server_env
|
|
- MAVEN_PROJECTS='server'
|
|
before_script: *setup_java_test
|
|
script: *run_java_test
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_server_module
|
|
name: "(openjdk11) server module test"
|
|
jdk: openjdk11
|
|
|
|
- &test_server_module_sqlcompat
|
|
name: "(openjdk8) server module test (SQL Compatibility)"
|
|
env: *server_env
|
|
before_script: *setup_java_test
|
|
script: *run_java_sql_compat_test
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_server_module_sqlcompat
|
|
name: "(openjdk11) server module test (SQL Compatibility)"
|
|
jdk: openjdk11
|
|
|
|
- &test_modules
|
|
name: "(openjdk8) other modules test"
|
|
env: &other_env
|
|
- MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console'
|
|
before_script: *setup_java_test
|
|
script: *run_java_test
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_modules
|
|
name: "(openjdk11) other modules test"
|
|
jdk: openjdk11
|
|
|
|
- &test_modules_sqlcompat
|
|
name: "(openjdk8) other modules test (SQL Compatibility)"
|
|
env: *other_env
|
|
before_script: *setup_java_test
|
|
script: *run_java_sql_compat_test
|
|
after_success: *upload_java_unit_test_coverage
|
|
|
|
- <<: *test_modules_sqlcompat
|
|
name: "(openjdk11) other modules test (SQL Compatibility)"
|
|
jdk: openjdk11
|
|
|
|
- &test_webconsole
|
|
name: "web console"
|
|
install: skip
|
|
script:
|
|
- ${MVN} test -pl 'web-console'
|
|
after_success:
|
|
- (cd web-console && travis_retry npm run codecov) # retry in case of network error
|
|
|
|
- name: "docs"
|
|
install: (cd website && npm install)
|
|
script: |-
|
|
(cd website && npm run lint && npm run spellcheck) || { echo "
|
|
|
|
If there are spell check errors:
|
|
|
|
1) Suppressing False Positives: Edit website/.spelling to add suppressions. Instructions
|
|
are at the top of the file and explain how to suppress false positives either globally or
|
|
within a particular file.
|
|
|
|
2) Running Spell Check Locally: cd website && npm install && npm run spellcheck
|
|
|
|
For more information, refer to: https://www.npmjs.com/package/markdown-spellcheck
|
|
|
|
" && false; }
|
|
|
|
# Integration tests Java Compile version is set by the machine environment jdk (set by the jdk key)
|
|
# Integration tests Java Runtime version is set by the JVM_RUNTIME env property (set env key to -Djvm.runtime=<JVM_RUNTIME_VERSION>)
|
|
# (Currently integration tests only support running with jvm runtime 8 and 11)
|
|
# START - Integration tests for Compile with Java 8 and Run with Java 8
|
|
- &integration_batch_index
|
|
name: "(Compile=openjdk8, Run=openjdk8) batch index integration test"
|
|
jdk: openjdk8
|
|
services: &integration_test_services
|
|
- docker
|
|
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: &run_integration_test
|
|
- ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} ${MAVEN_SKIP}
|
|
after_failure: &integration_test_diags
|
|
- for v in ~/shared/logs/*.log ; do
|
|
echo $v logtail ======================== ; tail -100 $v ;
|
|
done
|
|
- for v in broker middlemanager overlord router coordinator historical ; do
|
|
echo $v dmesg ======================== ;
|
|
docker exec -it druid-$v sh -c 'dmesg | tail -3' ;
|
|
done
|
|
|
|
- &integration_perfect_rollup_parallel_batch_index
|
|
name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test"
|
|
jdk: openjdk8
|
|
services: *integration_test_services
|
|
env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: *run_integration_test
|
|
after_failure: *integration_test_diags
|
|
|
|
- &integration_kafka_index
|
|
name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test"
|
|
jdk: openjdk8
|
|
services: *integration_test_services
|
|
env: TESTNG_GROUPS='-Dgroups=kafka-index' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: *run_integration_test
|
|
after_failure: *integration_test_diags
|
|
|
|
- &integration_query
|
|
name: "(Compile=openjdk8, Run=openjdk8) query integration test"
|
|
jdk: openjdk8
|
|
services: *integration_test_services
|
|
env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: *run_integration_test
|
|
after_failure: *integration_test_diags
|
|
|
|
- &integration_security
|
|
name: "(Compile=openjdk8, Run=openjdk8) security integration test"
|
|
jdk: openjdk8
|
|
services: *integration_test_services
|
|
env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: *run_integration_test
|
|
after_failure: *integration_test_diags
|
|
|
|
- &integration_realtime_index
|
|
name: "(Compile=openjdk8, Run=openjdk8) realtime index integration test"
|
|
jdk: openjdk8
|
|
services: *integration_test_services
|
|
env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: *run_integration_test
|
|
after_failure: *integration_test_diags
|
|
|
|
- &integration_tests
|
|
name: "(Compile=openjdk8, Run=openjdk8) other integration test"
|
|
jdk: openjdk8
|
|
services: *integration_test_services
|
|
env: TESTNG_GROUPS='-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage' JVM_RUNTIME='-Djvm.runtime=8'
|
|
script: *run_integration_test
|
|
after_failure: *integration_test_diags
|
|
# END - Integration tests for Compile with Java 8 and Run with Java 8
|
|
|
|
# START - Integration tests for Compile with Java 8 and Run with Java 11
|
|
- <<: *integration_batch_index
|
|
name: "(Compile=openjdk8, Run=openjdk11) batch index integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=11'
|
|
|
|
- <<: *integration_perfect_rollup_parallel_batch_index
|
|
name: "(Compile=openjdk8, Run=openjdk11) perfect rollup parallel batch index integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=11'
|
|
|
|
- <<: *integration_kafka_index
|
|
name: "(Compile=openjdk8, Run=openjdk11) kafka index integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-Dgroups=kafka-index' JVM_RUNTIME='-Djvm.runtime=11'
|
|
|
|
- <<: *integration_query
|
|
name: "(Compile=openjdk8, Run=openjdk11) query integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=11'
|
|
|
|
- <<: *integration_security
|
|
name: "(Compile=openjdk8, Run=openjdk11) security integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=11'
|
|
|
|
- <<: *integration_realtime_index
|
|
name: "(Compile=openjdk8, Run=openjdk11) realtime index integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=11'
|
|
|
|
- <<: *integration_tests
|
|
name: "(Compile=openjdk8, Run=openjdk11) other integration test"
|
|
jdk: openjdk8
|
|
env: TESTNG_GROUPS='-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage' JVM_RUNTIME='-Djvm.runtime=11'
|
|
# END - Integration tests for Compile with Java 8 and Run with Java 11
|
|
|
|
- name: "security vulnerabilities"
|
|
stage: cron
|
|
install: skip
|
|
script: |-
|
|
${MVN} dependency-check:check || { echo "
|
|
|
|
The OWASP dependency check has found security vulnerabilities. Please use a newer version
|
|
of the dependency that does not have vulnerabilities. If the analysis has false positives,
|
|
they can be suppressed by adding entries to owasp-dependency-check-suppressions.xml (for more
|
|
information, see https://jeremylong.github.io/DependencyCheck/general/suppression.html).
|
|
|
|
" && false; }
|
|
|
|
# Travis CI only supports per build (and not per-job notifications): https://github.com/travis-ci/travis-ci/issues/9888
|
|
notifications:
|
|
email:
|
|
if: type = cron
|
|
recipients:
|
|
# This is the string "dev@druid.apache.org" encrypted against the apache/druid repo so that forks are unable to
|
|
# use this notification:
|
|
# https://github.com/travis-ci/travis-ci/issues/1094#issuecomment-215019909
|
|
# https://github.com/travis-ci/travis-ci/issues/2711
|
|
- secure: "MupjX/0jLwh3XzHPl74BTk2/Kp5r+8TrEewfRhpQdWKFMBXLKNqu0k2VXf5C/NIg3uvPianq3REk+qeTHI8dL2ShjiWS/eIRkJOHLfObdNNBuos5fo4TxAuBQcXyT4VjAq5jnAkH84Pxf2Nl0rkisWoIhvwSX7+kNrjW1qdu7K0="
|
|
on_success: change
|
|
on_failure: change
|