From d242a9314b8a4979e5233f1b3d4ac76267bdc4e1 Mon Sep 17 00:00:00 2001 From: Tejaswini Bandlamudi <96047043+tejaswini-imply@users.noreply.github.com> Date: Wed, 9 Nov 2022 14:50:31 +0530 Subject: [PATCH] Adds license and security vulnerabilities checks for Hadoop3 build (#13270) * adds license and security vulnerabilities check for Hadoop3 builds * spacing * fixes bugs * updates check_test_suite.py to always run license checks with Hadoop3 * nit * run analyze dependencies, analyze hadoop 3 dependencies * run tests * revert analyze dependencies, analyze hadoop 3 dependencies addition in check_test_suite.py * fixes bug * revert code change --- .travis.yml | 51 +++++++++++++++++++++------------------------ check_test_suite.py | 2 +- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index efee82a8625..81f906f72cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,8 @@ jobs: - name: "spotbugs checks" script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' - - name: "license checks" + - &license_checks + name: "license checks" before_script: &setup_generate_license - sudo apt-get update && sudo apt-get install python3 -y - curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 @@ -100,12 +101,18 @@ jobs: - > ${MVN} apache-rat:check -Prat --fail-at-end -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - -Drat.consoleOutput=true + -Drat.consoleOutput=true ${HADOOP_PROFILE} # Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available # (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system). - mkdir -p target - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 - distribution/bin/check-licenses.py licenses.yaml target/license-reports + + - <<: *license_checks + name: "license checks with Hadoop3" + env: + - HADOOP_PROFILE='-Phadoop3' + - name: "script checks" install: skip # who watches the watchers? @@ -120,9 +127,10 @@ jobs: ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - - name: "analyze dependencies" + - &analyze_dependencies + name: "analyze dependencies" script: |- - MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo " + MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo " The dependency analysis has found a dependency that is either: @@ -142,27 +150,10 @@ jobs: " && false; } - - name: "analyze hadoop 3 dependencies" - script: |- - MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true -Phadoop3 || { echo " - - The dependency analysis has found a dependency that is either: - - 1) Used and undeclared: These are available as a transitive dependency but should be explicitly - added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is - shown above. - - 2) Unused and declared: These are not needed and removing them from the POM will speed up the build - and reduce the artifact size. The dependencies to remove are shown above. - - If there are false positive dependency analysis warnings, they can be suppressed: - https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies - https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html - - For more information, refer to: - https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html - - " && false; } + - <<: *analyze_dependencies + name: "analyze hadoop 3 dependencies" + env: + - HADOOP_PROFILE='-Phadoop3' - name: "intellij inspections" script: > @@ -803,11 +794,12 @@ jobs: echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0; done - - name: "security vulnerabilities" + - &security_vulnerabilities + name: "security vulnerabilities" stage: cron install: skip script: |- - ${MVN} dependency-check:purge dependency-check:check || { echo " + ${MVN} dependency-check:purge dependency-check:check ${HADOOP_PROFILE} || { echo " The OWASP dependency check has found security vulnerabilities. Please use a newer version of the dependency that does not have vulnerabilities. To see a report run @@ -818,6 +810,11 @@ jobs: " && false; } + - <<: *security_vulnerabilities + name: "security vulnerabilities with Hadoop3" + env: + - HADOOP_PROFILE='-Phadoop3' + # Travis CI only supports per build (and not per-job notifications): https://github.com/travis-ci/travis-ci/issues/9888 notifications: email: diff --git a/check_test_suite.py b/check_test_suite.py index 8fd73911142..741b14180aa 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -22,7 +22,7 @@ import sys # this script does some primitive examination of git diff to determine if a test suite needs to be run or not # these jobs should always be run, no matter what -always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check'] +always_run_jobs = ['license checks', 'license checks with Hadoop3', '(openjdk8) packaging check', '(openjdk11) packaging check'] # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed