From 7b7ec5687e867877f3d2e7050da27b504fc623ee Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 11 Mar 2021 10:56:07 +0900 Subject: [PATCH] HADOOP-16870. Use spotbugs-maven-plugin instead of findbugs-maven-plugin (#2753) Removed findbugs from the hadoop build images and added spotbugs instead. Upgraded SpotBugs to 4.2.2 and spotbugs-maven-plugin to 4.2.0. Reviewed-by: Masatake Iwasaki (cherry picked from commit 23b343aed17119b905dfa61ab1ce32ab554bfb57) Conflicts: dev-support/docker/Dockerfile hadoop-project/pom.xml (cherry picked from commit de2904f123a7e3a56e4f8572f24b2e75ea606e80) Conflicts: dev-support/docker/Dockerfile_aarch64 hadoop-cloud-storage-project/hadoop-cos/pom.xml hadoop-common-project/hadoop-registry/pom.xml hadoop-project/pom.xml (cherry picked from commit 0b278502e25e7a5d52bc9c2873058413f77dd7fa) Conflicts: dev-support/docker/Dockerfile hadoop-project/pom.xml --- BUILDING.txt | 2 +- dev-support/Jenkinsfile | 6 ++-- dev-support/bin/hadoop.sh | 4 +-- dev-support/docker/Dockerfile | 28 +++++++++++-------- hadoop-common-project/hadoop-auth/pom.xml | 4 +-- hadoop-common-project/hadoop-kms/pom.xml | 4 +-- hadoop-common-project/hadoop-minikdc/pom.xml | 4 +-- hadoop-common-project/hadoop-nfs/pom.xml | 4 +-- .../hadoop-hdfs-httpfs/pom.xml | 4 +-- .../hadoop-mapreduce-client/pom.xml | 5 ++-- .../hadoop-mapreduce-examples/pom.xml | 5 ++-- hadoop-mapreduce-project/pom.xml | 12 +++----- hadoop-project-dist/pom.xml | 5 ++-- hadoop-project/pom.xml | 17 ----------- hadoop-tools/hadoop-aliyun/pom.xml | 5 ++-- hadoop-tools/hadoop-archive-logs/pom.xml | 5 ++-- hadoop-tools/hadoop-aws/pom.xml | 5 ++-- hadoop-tools/hadoop-azure/pom.xml | 5 ++-- hadoop-tools/hadoop-datajoin/pom.xml | 5 ++-- hadoop-tools/hadoop-fs2img/pom.xml | 5 ++-- hadoop-tools/hadoop-gridmix/pom.xml | 5 ++-- hadoop-tools/hadoop-kafka/pom.xml | 5 ++-- hadoop-tools/hadoop-openstack/pom.xml | 5 ++-- hadoop-tools/hadoop-rumen/pom.xml | 5 ++-- hadoop-tools/hadoop-sls/pom.xml | 5 ++-- hadoop-tools/hadoop-streaming/pom.xml | 5 ++-- .../pom.xml | 4 +-- hadoop-yarn-project/hadoop-yarn/pom.xml | 5 ++-- hadoop-yarn-project/pom.xml | 7 ++--- pom.xml | 18 ++++++++++++ 30 files changed, 91 insertions(+), 107 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 236f5187ae8..495693b79f9 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -114,7 +114,7 @@ Maven build goals: * Compile : mvn compile [-Pnative] * Run tests : mvn test [-Pnative] [-Pshelltest] * Create JAR : mvn package - * Run findbugs : mvn compile findbugs:findbugs + * Run spotbugs : mvn compile spotbugs:spotbugs * Run checkstyle : mvn compile checkstyle:checkstyle * Install JAR in M2 cache : mvn install * Deploy JAR to Maven repo : mvn deploy diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 411228c0d0c..ebd6ed4f03f 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -120,11 +120,11 @@ pipeline { YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--dockermemlimit=22g") - # -1 findbugs issues that show up prior to the patch being applied - YETUS_ARGS+=("--findbugs-strict-precheck") + # -1 spotbugs issues that show up prior to the patch being applied + YETUS_ARGS+=("--spotbugs-strict-precheck") # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") + YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") # URL for user-side presentation in reports and such to our artifacts # (needs to match the archive bits below) diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh index 3343014aae8..beebea8c97f 100755 --- a/dev-support/bin/hadoop.sh +++ b/dev-support/bin/hadoop.sh @@ -482,7 +482,7 @@ function personality_file_tests fi if [[ ${filename} =~ \.java$ ]]; then - add_test findbugs + add_test spotbugs fi } @@ -550,7 +550,7 @@ function shadedclient_rebuild echo_and_redirect "${logfile}" \ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ "${modules[@]}" \ - -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true + -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true count=$("${GREP}" -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index d0b51e7f7a1..14f63240fab 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -84,6 +84,23 @@ RUN apt-get -q update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME /usr +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 + +####### +# Install SpotBugs 4.2.2 +####### +RUN mkdir -p /opt/spotbugs \ + && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz \ + && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ + && chmod +x /opt/spotbugs/bin/* +ENV SPOTBUGS_HOME /opt/spotbugs + ####### # OpenJDK 8 ####### @@ -122,17 +139,6 @@ ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -###### -# Install findbugs 3.1.0 (3.1.0 ships with Bionic) -# Ant is needed for findbugs -###### -# hadolint ignore=DL3008 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends findbugs ant \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -ENV FINDBUGS_HOME /usr - # Install pylint and python-dateutil RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index a321791007d..f510da5c8f4 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -228,8 +228,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin ${basedir}/dev-support/findbugsExcludeFile.xml diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index eb94687e48f..f31986235be 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -235,8 +235,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin ${basedir}/dev-support/findbugsExcludeFile.xml diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml index 57fdd16e0c0..50e4643302b 100644 --- a/hadoop-common-project/hadoop-minikdc/pom.xml +++ b/hadoop-common-project/hadoop-minikdc/pom.xml @@ -53,8 +53,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin ${basedir}/dev-support/findbugsExcludeFile.xml diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index 1c27b00b091..c9288fd91ea 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -102,8 +102,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin ${basedir}/dev-support/findbugsExcludeFile.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index 5adda3f84b3..a593020a32c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -337,8 +337,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin ${basedir}/dev-support/findbugsExcludeFile.xml diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index b968ee0641e..fa293e3a803 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -162,10 +162,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${mr.basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index a41825f33fc..3fd87ec0439 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -133,10 +133,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${mr.examples.basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 5c2ebcdd74a..035ebd196a9 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -178,10 +178,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${mr.basedir}/dev-support/findbugs-exclude.xml Max @@ -299,12 +298,9 @@ - org.codehaus.mojo - findbugs-maven-plugin - - 2.3.2 + com.github.spotbugs + spotbugs-maven-plugin - true true diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 85f0020b9f3..c4fded3be50 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -90,11 +90,10 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin ${basedir}/dev-support/findbugsExcludeFile.xml - true 2048 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index ee71a276982..2459490b63c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -90,7 +90,6 @@ 3.4.13 2.13.0 3.0.5 - 3.1.0-RC1 2.1.7 27.0-jre @@ -1495,18 +1494,6 @@ maven-war-plugin ${maven-war-plugin.version} - - org.codehaus.mojo - findbugs-maven-plugin - ${findbugs.version} - - - com.github.spotbugs - spotbugs - ${spotbugs.version} - - - org.codehaus.mojo native-maven-plugin @@ -1611,10 +1598,6 @@ - - org.codehaus.mojo - findbugs-maven-plugin - org.apache.maven.plugins maven-remote-resources-plugin diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml index 759baeb68a1..c20667fe8f3 100644 --- a/hadoop-tools/hadoop-aliyun/pom.xml +++ b/hadoop-tools/hadoop-aliyun/pom.xml @@ -58,10 +58,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml index e943860e824..b0c5664ebe2 100644 --- a/hadoop-tools/hadoop-archive-logs/pom.xml +++ b/hadoop-tools/hadoop-archive-logs/pom.xml @@ -194,10 +194,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index bc8441624a3..55f4c454b97 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -320,10 +320,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 390f32aba84..6f89d319dfe 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -50,10 +50,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml index 35adad68f6e..3f1af900b2a 100644 --- a/hadoop-tools/hadoop-datajoin/pom.xml +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -108,10 +108,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml index 90f27347e5a..3b2a37be4ea 100644 --- a/hadoop-tools/hadoop-fs2img/pom.xml +++ b/hadoop-tools/hadoop-fs2img/pom.xml @@ -88,10 +88,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index 9dee44534ae..197d144f7aa 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -113,10 +113,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml index 98a26159a8f..94bbb76dad2 100644 --- a/hadoop-tools/hadoop-kafka/pom.xml +++ b/hadoop-tools/hadoop-kafka/pom.xml @@ -39,10 +39,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true Max diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml index b6c7fdbb173..827c1f658ac 100644 --- a/hadoop-tools/hadoop-openstack/pom.xml +++ b/hadoop-tools/hadoop-openstack/pom.xml @@ -66,10 +66,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index d18574c8ba3..a80c3e13d5c 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -102,10 +102,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index c823e1c94af..e7c6094c037 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -108,10 +108,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 829f0911e5c..143c150af71 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -119,10 +119,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index 07315669f26..7d989135035 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -360,8 +360,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin true diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index 50599f26a05..55ba4adcb67 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -39,10 +39,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${yarn.basedir}/dev-support/findbugs-exclude.xml Max diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index cea5a725b3c..46b67b92273 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -204,12 +204,9 @@ - org.codehaus.mojo - findbugs-maven-plugin - - 2.3.2 + com.github.spotbugs + spotbugs-maven-plugin - true true diff --git a/pom.xml b/pom.xml index d6821fb6480..fc36592add6 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x 3.1.0 8.26 1.4.3 + 4.2.2 + 4.2.0 bash @@ -337,6 +339,18 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x dependency-check-maven ${dependency-check-maven.version} + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + @@ -418,6 +432,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x dependency-check-maven ${dependency-check-maven.version} + + com.github.spotbugs + spotbugs-maven-plugin +