Adds license and security vulnerabilities checks for Hadoop3 build (#13270)

* adds license and security vulnerabilities check for Hadoop3 builds

* spacing

* fixes bugs

* updates check_test_suite.py to always run license checks with Hadoop3

* nit

* run analyze dependencies, analyze hadoop 3 dependencies

* run tests

* revert analyze dependencies, analyze hadoop 3 dependencies addition in check_test_suite.py

* fixes bug

* revert code change

.travis.yml

@@ -89,7 +89,8 @@ jobs:
     - name: "spotbugs checks"
       script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
 
-    - name: "license checks"
+    - &license_checks
+      name: "license checks"
       before_script: &setup_generate_license
         - sudo apt-get update && sudo apt-get install python3 -y
         - curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
@@ -100,12 +101,18 @@ jobs:
         - >
           ${MVN} apache-rat:check -Prat --fail-at-end
           -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
-          -Drat.consoleOutput=true
+          -Drat.consoleOutput=true ${HADOOP_PROFILE}
         # Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available
         # (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system).
         - mkdir -p target
         - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
         - distribution/bin/check-licenses.py licenses.yaml target/license-reports
+
+    - <<: *license_checks
+      name: "license checks with Hadoop3"
+      env:
+        - HADOOP_PROFILE='-Phadoop3'
+
     - name: "script checks"
       install: skip
       # who watches the watchers?
@@ -120,9 +127,10 @@ jobs:
         ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -DstrictCompile compile test-compile --fail-at-end
         ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
-    - name: "analyze dependencies"
+    - &analyze_dependencies
+      name: "analyze dependencies"
       script: |-
-        MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo "
+        MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo "
 
         The dependency analysis has found a dependency that is either:
 
@@ -142,27 +150,10 @@ jobs:
 
         " && false; }
 
-    - name: "analyze hadoop 3 dependencies"
-      script: |-
-        MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true -Phadoop3 || { echo "
-
-        The dependency analysis has found a dependency that is either:
-
-        1) Used and undeclared: These are available as a transitive dependency but should be explicitly
-        added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is
-        shown above.
-
-        2) Unused and declared: These are not needed and removing them from the POM will speed up the build
-        and reduce the artifact size. The dependencies to remove are shown above.
-
-        If there are false positive dependency analysis warnings, they can be suppressed:
-        https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
-        https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
-
-        For more information, refer to:
-        https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
-
-        " && false; }
+    - <<: *analyze_dependencies
+      name: "analyze hadoop 3 dependencies"
+      env:
+        - HADOOP_PROFILE='-Phadoop3'
 
     - name: "intellij inspections"
       script: >
@@ -803,11 +794,12 @@ jobs:
           echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";
           sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0;
           done
-    - name: "security vulnerabilities"
+    - &security_vulnerabilities
+      name: "security vulnerabilities"
       stage: cron
       install: skip
       script: |-
-        ${MVN} dependency-check:purge dependency-check:check || { echo "
+        ${MVN} dependency-check:purge dependency-check:check ${HADOOP_PROFILE} || { echo "
 
         The OWASP dependency check has found security vulnerabilities. Please use a newer version
         of the dependency that does not have vulnerabilities. To see a report run
@@ -818,6 +810,11 @@ jobs:
 
         " && false; }
 
+    - <<: *security_vulnerabilities
+      name: "security vulnerabilities with Hadoop3"
+      env:
+        - HADOOP_PROFILE='-Phadoop3'
+
 # Travis CI only supports per build (and not per-job notifications): https://github.com/travis-ci/travis-ci/issues/9888
 notifications:
   email:

check_test_suite.py

@@ -22,7 +22,7 @@ import sys
 # this script does some primitive examination of git diff to determine if a test suite needs to be run or not
 
 # these jobs should always be run, no matter what
-always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check']
+always_run_jobs = ['license checks', 'license checks with Hadoop3', '(openjdk8) packaging check', '(openjdk11) packaging check']
 
 # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all
 # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed