diff --git a/build.gradle b/build.gradle index c0fd415abed..a043c4789f0 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,7 @@ plugins { id "base" id "com.palantir.consistent-versions" version "1.12.4" id "com.gradle.build-scan" version "3.0" + id 'de.thetaphi.forbiddenapis' version '2.7' apply false } // Project version and main properties. Applies to all projects. @@ -33,6 +34,9 @@ apply from: file('gradle/maven/defaults-maven.gradle') // IDE settings and specials. apply from: file('gradle/defaults-idea.gradle') +// Validation tasks +apply from: file('gradle/validation/forbidden-apis.gradle') + // Additional development aids. apply from: file('gradle/maven/maven-local.gradle') apply from: file('gradle/testing/per-project-summary.gradle') @@ -48,4 +52,5 @@ apply from: file('gradle/ant-compat/resolve.gradle') apply from: file('gradle/ant-compat/post-jar.gradle') apply from: file('gradle/ant-compat/test-classes-cross-deps.gradle') apply from: file('gradle/ant-compat/artifact-naming.gradle') +apply from: file('gradle/ant-compat/solr-forbidden-apis.gradle') diff --git a/gradle/ant-compat/solr-forbidden-apis.gradle b/gradle/ant-compat/solr-forbidden-apis.gradle new file mode 100644 index 00000000000..54ffd5d1fce --- /dev/null +++ b/gradle/ant-compat/solr-forbidden-apis.gradle @@ -0,0 +1,9 @@ + +// Why does solr exclude these from forbidden API checks? + +configure(project(":solr:core")) { + configure([forbiddenApisMain, forbiddenApisTest]) { + exclude "org/apache/solr/internal/**" + exclude "org/apache/hadoop/**" + } +} \ No newline at end of file diff --git a/gradle/help.gradle b/gradle/help.gradle index 441d66644ec..0ee82ac18fa 100644 --- a/gradle/help.gradle +++ b/gradle/help.gradle @@ -5,6 +5,7 @@ configure(rootProject) { ["Workflow", "help/workflow.txt", "Typical workflow commands."], ["Ant", "help/ant.txt", "Ant-gradle migration help."], ["Tests", "help/tests.txt", "Tests, filtering, beasting, etc."], + ["ForbiddenApis", "help/forbiddenApis.txt", "How to add/apply rules for forbidden APIs."], ] helpFiles.each { section, path, sectionInfo -> diff --git a/gradle/validation/forbidden-apis.gradle b/gradle/validation/forbidden-apis.gradle new file mode 100644 index 00000000000..80cd33dbcfd --- /dev/null +++ b/gradle/validation/forbidden-apis.gradle @@ -0,0 +1,106 @@ +// This configures application of forbidden API rules +// via https://github.com/policeman-tools/forbidden-apis + +// Only apply forbidden-apis to java projects. +allprojects { prj -> + plugins.withId("java", { + prj.apply plugin: 'de.thetaphi.forbiddenapis' + + // This helper method appends signature files based on a set of true + // dependencies from a given configuration. + def dynamicSignatures = { configuration, suffix -> + def deps = configuration.resolvedConfiguration.resolvedArtifacts + .collect { a -> a.moduleVersion.id } + .collect { id -> [ + "${id.group}.${id.name}.all.txt", + "${id.group}.${id.name}.${suffix}.txt", + ]} + .flatten() + .sort() + + deps += ["defaults.all.txt", "defaults.${suffix}.txt"] + + deps.each { sig -> + def signaturesFile = rootProject.file("gradle/validation/forbidden-apis/${sig}") + if (signaturesFile.exists()) { + logger.info("Signature file applied: ${sig}") + signaturesFiles += files(signaturesFile) + } else { + logger.debug("Signature file omitted (does not exist): ${sig}") + } + } + } + + // Configure defaults for sourceSets.main + forbiddenApisMain { + bundledSignatures += [ + 'jdk-unsafe', + 'jdk-deprecated', + 'jdk-non-portable', + 'jdk-reflection', + 'jdk-system-out', + ] + + suppressAnnotations += [ + "**.SuppressForbidden" + ] + } + + // Configure defaults for sourceSets.test + forbiddenApisTest { + bundledSignatures += [ + 'jdk-unsafe', + 'jdk-deprecated', + 'jdk-non-portable', + 'jdk-reflection', + ] + + signaturesFiles = files( + rootProject.file("gradle/validation/forbidden-apis/defaults.tests.txt") + ) + + suppressAnnotations += [ + "**.SuppressForbidden" + ] + } + + // Attach validation to check task. + check.dependsOn forbiddenApisMain + check.dependsOn forbiddenApisTest + + // Disable sysout signatures for these projects. + if (prj.path in [ + ":lucene:demo", + ":lucene:benchmark", + ":lucene:test-framework", + ":solr:solr-ref-guide", + ":solr:test-framework" + ]) { + forbiddenApisMain.bundledSignatures -= [ + 'jdk-system-out' + ] + } + + // Configure lucene-specific rules. + if (prj.path.startsWith(":lucene")) { + forbiddenApisMain { + doFirst dynamicSignatures.curry(configurations.compileClasspath, "lucene") + } + + forbiddenApisTest { + doFirst dynamicSignatures.curry(configurations.testCompileClasspath, "lucene") + } + } + + // Configure solr-specific rules. + if (prj.path.startsWith(":solr")) { + forbiddenApisMain { + doFirst dynamicSignatures.curry(configurations.compileClasspath, "solr") + } + + forbiddenApisTest { + doFirst dynamicSignatures.curry(configurations.testCompileClasspath, "solr") + } + } + }) +} \ No newline at end of file diff --git a/gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt b/gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt new file mode 100644 index 00000000000..fd54b9ea1f7 --- /dev/null +++ b/gradle/validation/forbidden-apis/com.carrotsearch.randomizedtesting.randomizedtesting-runner.all.txt @@ -0,0 +1 @@ +com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds diff --git a/gradle/validation/forbidden-apis/com.fasterxml.jackson.core.jackson-annotations.solr.txt b/gradle/validation/forbidden-apis/com.fasterxml.jackson.core.jackson-annotations.solr.txt new file mode 100644 index 00000000000..af88dea6791 --- /dev/null +++ b/gradle/validation/forbidden-apis/com.fasterxml.jackson.core.jackson-annotations.solr.txt @@ -0,0 +1,2 @@ +@defaultMessage Use org.apache.solr.common.annotation.JsonProperty instead +com.fasterxml.jackson.annotation.JsonProperty diff --git a/gradle/validation/forbidden-apis/com.google.guava.guava.all.txt b/gradle/validation/forbidden-apis/com.google.guava.guava.all.txt new file mode 100644 index 00000000000..4e04e789415 --- /dev/null +++ b/gradle/validation/forbidden-apis/com.google.guava.guava.all.txt @@ -0,0 +1,17 @@ +@defaultMessage Use corresponding Java 8 functional/streaming interfaces +com.google.common.base.Function +com.google.common.base.Joiner +com.google.common.base.Predicate +com.google.common.base.Supplier + +@defaultMessage Use java.nio.charset.StandardCharsets instead +com.google.common.base.Charsets + +@defaultMessage Use methods in java.util.Objects instead +com.google.common.base.Objects#equal(java.lang.Object,java.lang.Object) +com.google.common.base.Objects#hashCode(java.lang.Object[]) +com.google.common.base.Preconditions#checkNotNull(java.lang.Object) +com.google.common.base.Preconditions#checkNotNull(java.lang.Object,java.lang.Object) + +@defaultMessage Use methods in java.util.Comparator instead +com.google.common.collect.Ordering diff --git a/gradle/validation/forbidden-apis/commons-codec.commons-codec.all.txt b/gradle/validation/forbidden-apis/commons-codec.commons-codec.all.txt new file mode 100644 index 00000000000..b0efb8e78d2 --- /dev/null +++ b/gradle/validation/forbidden-apis/commons-codec.commons-codec.all.txt @@ -0,0 +1,2 @@ +@defaultMessage Use java.nio.charset.StandardCharsets instead +org.apache.commons.codec.Charsets diff --git a/gradle/validation/forbidden-apis/defaults.all.txt b/gradle/validation/forbidden-apis/defaults.all.txt new file mode 100644 index 00000000000..0a81d03e8a3 --- /dev/null +++ b/gradle/validation/forbidden-apis/defaults.all.txt @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@defaultMessage Spawns threads with vague names; use a custom thread factory (Lucene's NamedThreadFactory, Solr's DefaultSolrThreadFactory) and name threads so that you can tell (by its name) which executor it is associated with +java.util.concurrent.Executors#newFixedThreadPool(int) +java.util.concurrent.Executors#newSingleThreadExecutor() +java.util.concurrent.Executors#newCachedThreadPool() +java.util.concurrent.Executors#newSingleThreadScheduledExecutor() +java.util.concurrent.Executors#newScheduledThreadPool(int) +java.util.concurrent.Executors#defaultThreadFactory() +java.util.concurrent.Executors#privilegedThreadFactory() + +@defaultMessage Properties files should be read/written with Reader/Writer, using UTF-8 charset. This allows reading older files with unicode escapes, too. +java.util.Properties#load(java.io.InputStream) +java.util.Properties#save(java.io.OutputStream,java.lang.String) +java.util.Properties#store(java.io.OutputStream,java.lang.String) + +@defaultMessage The context classloader should never be used for resource lookups, unless there is a 3rd party library that needs it. Always pass a classloader down as method parameters. +java.lang.Thread#getContextClassLoader() +java.lang.Thread#setContextClassLoader(java.lang.ClassLoader) + +java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars +java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars + +java.io.File#delete() @ use Files.delete for real exception, IOUtils.deleteFilesIgnoringExceptions if you dont care + +java.util.Collections#shuffle(java.util.List) @ Use shuffle(List, Random) instead so that it can be reproduced + +java.util.Locale#forLanguageTag(java.lang.String) @ use new Locale.Builder().setLanguageTag(...).build() which has error handling +java.util.Locale#toString() @ use Locale#toLanguageTag() for a standardized BCP47 locale name + +@defaultMessage Constructors for wrapper classes of Java primitives should be avoided in favor of the public static methods available or autoboxing +java.lang.Integer#(int) +java.lang.Integer#(java.lang.String) +java.lang.Byte#(byte) +java.lang.Byte#(java.lang.String) +java.lang.Short#(short) +java.lang.Short#(java.lang.String) +java.lang.Long#(long) +java.lang.Long#(java.lang.String) +java.lang.Boolean#(boolean) +java.lang.Boolean#(java.lang.String) +java.lang.Character#(char) +java.lang.Float#(float) +java.lang.Float#(double) +java.lang.Float#(java.lang.String) +java.lang.Double#(double) +java.lang.Double#(java.lang.String) diff --git a/gradle/validation/forbidden-apis/defaults.lucene.txt b/gradle/validation/forbidden-apis/defaults.lucene.txt new file mode 100644 index 00000000000..0cc4edda6bd --- /dev/null +++ b/gradle/validation/forbidden-apis/defaults.lucene.txt @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@defaultMessage Use NIO.2 instead +java.io.File +java.io.FileInputStream +java.io.FileOutputStream +java.io.PrintStream#(java.lang.String,java.lang.String) +java.io.PrintWriter#(java.lang.String,java.lang.String) +java.util.Formatter#(java.lang.String,java.lang.String,java.util.Locale) +java.io.RandomAccessFile +java.nio.file.Path#toFile() +java.util.jar.JarFile +java.util.zip.ZipFile +@defaultMessage Prefer using ArrayUtil as Arrays#copyOfRange fills zeros for bad bounds +java.util.Arrays#copyOfRange(byte[],int,int) +java.util.Arrays#copyOfRange(char[],int,int) +java.util.Arrays#copyOfRange(short[],int,int) +java.util.Arrays#copyOfRange(int[],int,int) +java.util.Arrays#copyOfRange(long[],int,int) +java.util.Arrays#copyOfRange(float[],int,int) +java.util.Arrays#copyOfRange(double[],int,int) +java.util.Arrays#copyOfRange(boolean[],int,int) +java.util.Arrays#copyOfRange(java.lang.Object[],int,int) +java.util.Arrays#copyOfRange(java.lang.Object[],int,int,java.lang.Class) + +@defaultMessage Prefer using ArrayUtil as Arrays#copyOf fills zeros for bad bounds +java.util.Arrays#copyOf(byte[],int) +java.util.Arrays#copyOf(char[],int) +java.util.Arrays#copyOf(short[],int) +java.util.Arrays#copyOf(int[],int) +java.util.Arrays#copyOf(long[],int) +java.util.Arrays#copyOf(float[],int) +java.util.Arrays#copyOf(double[],int) +java.util.Arrays#copyOf(boolean[],int) +java.util.Arrays#copyOf(java.lang.Object[],int) +java.util.Arrays#copyOf(java.lang.Object[],int,java.lang.Class) diff --git a/gradle/validation/forbidden-apis/defaults.solr.txt b/gradle/validation/forbidden-apis/defaults.solr.txt new file mode 100644 index 00000000000..50c69ac0227 --- /dev/null +++ b/gradle/validation/forbidden-apis/defaults.solr.txt @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@defaultMessage Spawns threads without MDC logging context; use ExecutorUtil.newMDCAwareFixedThreadPool instead +java.util.concurrent.Executors#newFixedThreadPool(int,java.util.concurrent.ThreadFactory) + +@defaultMessage Spawns threads without MDC logging context; use ExecutorUtil.newMDCAwareSingleThreadExecutor instead +java.util.concurrent.Executors#newSingleThreadExecutor(java.util.concurrent.ThreadFactory) + +@defaultMessage Spawns threads without MDC logging context; use ExecutorUtil.newMDCAwareCachedThreadPool instead +java.util.concurrent.Executors#newCachedThreadPool(java.util.concurrent.ThreadFactory) + +@defaultMessage Use ExecutorUtil.MDCAwareThreadPoolExecutor instead of ThreadPoolExecutor +java.util.concurrent.ThreadPoolExecutor#(int,int,long,java.util.concurrent.TimeUnit,java.util.concurrent.BlockingQueue,java.util.concurrent.ThreadFactory,java.util.concurrent.RejectedExecutionHandler) +java.util.concurrent.ThreadPoolExecutor#(int,int,long,java.util.concurrent.TimeUnit,java.util.concurrent.BlockingQueue) +java.util.concurrent.ThreadPoolExecutor#(int,int,long,java.util.concurrent.TimeUnit,java.util.concurrent.BlockingQueue,java.util.concurrent.ThreadFactory) +java.util.concurrent.ThreadPoolExecutor#(int,int,long,java.util.concurrent.TimeUnit,java.util.concurrent.BlockingQueue,java.util.concurrent.RejectedExecutionHandler) + +@defaultMessage Use RTimer/TimeOut/System.nanoTime for time comparisons, and `new Date()` output/debugging/stats of timestamps. If for some miscellaneous reason, you absolutely need to use this, use a SuppressForbidden. +java.lang.System#currentTimeMillis() + +@defaultMessage Use slf4j classes instead +java.util.logging.** diff --git a/gradle/validation/forbidden-apis/defaults.tests.txt b/gradle/validation/forbidden-apis/defaults.tests.txt new file mode 100644 index 00000000000..02ab6c8dfd9 --- /dev/null +++ b/gradle/validation/forbidden-apis/defaults.tests.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +java.util.Random#() @ Use RandomizedRunner's random() instead +java.lang.Math#random() @ Use RandomizedRunner's random().nextDouble() instead + +# TODO: fix tests that do this! +#java.lang.System#currentTimeMillis() @ Don't depend on wall clock times +#java.lang.System#nanoTime() @ Don't depend on wall clock times + +@defaultMessage Use LuceneTestCase.collate instead, which can avoid JDK-8071862 +java.text.Collator#compare(java.lang.Object,java.lang.Object) +java.text.Collator#compare(java.lang.String,java.lang.String) diff --git a/gradle/validation/forbidden-apis/javax.servlet.javax.servlet-api.all.txt b/gradle/validation/forbidden-apis/javax.servlet.javax.servlet-api.all.txt new file mode 100644 index 00000000000..dc82e8fe45d --- /dev/null +++ b/gradle/validation/forbidden-apis/javax.servlet.javax.servlet-api.all.txt @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@defaultMessage Servlet API method is parsing request parameters without using the correct encoding if no extra configuration is given in the servlet container + +javax.servlet.ServletRequest#getParameter(java.lang.String) +javax.servlet.ServletRequest#getParameterMap() +javax.servlet.ServletRequest#getParameterNames() +javax.servlet.ServletRequest#getParameterValues(java.lang.String) + +javax.servlet.http.HttpServletRequest#getSession() @ Servlet API getter has side effect of creating sessions + +@defaultMessage Servlet API method is broken and slow in some environments (e.g., Jetty's UTF-8 readers) + +javax.servlet.ServletRequest#getReader() +javax.servlet.ServletResponse#getWriter() +javax.servlet.ServletInputStream#readLine(byte[],int,int) +javax.servlet.ServletOutputStream#print(boolean) +javax.servlet.ServletOutputStream#print(char) +javax.servlet.ServletOutputStream#print(double) +javax.servlet.ServletOutputStream#print(float) +javax.servlet.ServletOutputStream#print(int) +javax.servlet.ServletOutputStream#print(long) +javax.servlet.ServletOutputStream#print(java.lang.String) +javax.servlet.ServletOutputStream#println(boolean) +javax.servlet.ServletOutputStream#println(char) +javax.servlet.ServletOutputStream#println(double) +javax.servlet.ServletOutputStream#println(float) +javax.servlet.ServletOutputStream#println(int) +javax.servlet.ServletOutputStream#println(long) +javax.servlet.ServletOutputStream#println(java.lang.String) diff --git a/gradle/validation/forbidden-apis/junit.junit.lucene.txt b/gradle/validation/forbidden-apis/junit.junit.lucene.txt new file mode 100644 index 00000000000..fc8d20a63cc --- /dev/null +++ b/gradle/validation/forbidden-apis/junit.junit.lucene.txt @@ -0,0 +1 @@ +junit.framework.TestCase @ All classes should derive from LuceneTestCase diff --git a/gradle/validation/forbidden-apis/org.apache.logging.log4j.log4j-api.all.txt b/gradle/validation/forbidden-apis/org.apache.logging.log4j.log4j-api.all.txt new file mode 100644 index 00000000000..7816b651a9d --- /dev/null +++ b/gradle/validation/forbidden-apis/org.apache.logging.log4j.log4j-api.all.txt @@ -0,0 +1,3 @@ +@defaultMessage Use slf4j classes instead +org.apache.log4j.** +org.apache.logging.log4j.** diff --git a/help/forbiddenApis.txt b/help/forbiddenApis.txt new file mode 100644 index 00000000000..10f851924db --- /dev/null +++ b/help/forbiddenApis.txt @@ -0,0 +1,34 @@ +Forbidden API rules +=================== + +Uwe's excellent forbidden API checker is applied as part of 'check' +task. The rules for each project are sourced dynamically based on the +actual set of dependencies. + +If a given project has a dependency on an artifact called "foo.bar:baz" +then all of these rule files will be applied (all paths relative +to: gradle/validation/forbidden-apis/). + +defaults.all.txt +defaults.[project].txt +foo.bar.baz.all.txt +foo.bar.baz.[project].txt + +Note that the "defaults" can't reference any JARs other than Java's +runtime. + +Example +------- + +We'd like to prevent people from using Guava's +com.google.common.base.Charsets class. The rule would be: + +@defaultMessage Use java.nio.charset.StandardCharsets instead +com.google.common.base.Charsets + +and we would place this rule in this file: + +gradle/validation/forbidden-apis/com.google.guava.guava.all.txt + +From now on, if *any* module depends on this library, it will +automatically pick up the rule and enforce it.