From 2cc611604f16cbedf94fb218da9fe29da7727868 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 28 Aug 2018 10:03:30 +0300 Subject: [PATCH] Run Third party audit with forbidden APIs CLI (part3/3) (#33052) The new implementation is functional equivalent with the old, ant based one. It parses task standard error to get the missing classes and violations in the same way. I considered re-using ForbiddenApisCliTask but Gradle makes it hard to build inheritance with tasks that have task actions , since the order of the task actions can't be controlled. This inheritance isn't dully desired either as the third party audit task is much more opinionated and we don't want to expose some of the configuration. We could probably extract a common base class without any task actions, but probably more trouble than it's worth. Closes #31715 --- buildSrc/build.gradle | 1 - .../gradle/precommit/PrecommitTasks.groovy | 39 ++- .../precommit/ThirdPartyAuditTask.groovy | 297 ------------------ .../test/StandaloneRestTestPlugin.groovy | 2 + .../elasticsearch/gradle/JdkJarHellCheck.java | 81 +++++ .../precommit/ForbiddenApisCliTask.java | 49 ++- .../gradle/precommit/ThirdPartyAuditTask.java | 288 +++++++++++++++++ plugins/discovery-azure-classic/build.gradle | 2 +- plugins/discovery-ec2/build.gradle | 2 +- plugins/ingest-attachment/build.gradle | 22 +- plugins/repository-hdfs/build.gradle | 21 +- plugins/repository-s3/build.gradle | 2 +- server/build.gradle | 19 +- test/logger-usage/build.gradle | 11 +- x-pack/plugin/security/build.gradle | 4 +- x-pack/plugin/sql/sql-action/build.gradle | 11 +- x-pack/plugin/watcher/build.gradle | 2 +- 17 files changed, 505 insertions(+), 348 deletions(-) delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 967c2e27ee8..9918d54d707 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -102,7 +102,6 @@ dependencies { compile 'com.netflix.nebula:gradle-info-plugin:3.0.3' compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r' compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE.... - compile 'de.thetaphi:forbiddenapis:2.5' compile 'org.apache.rat:apache-rat:0.11' compile "org.elasticsearch:jna:4.5.1" compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4' diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index b63b1f40d80..d82302c8474 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -31,6 +31,11 @@ class PrecommitTasks { /** Adds a precommit task, which depends on non-test verification tasks. */ public static Task create(Project project, boolean includeDependencyLicenses) { + Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar") + project.dependencies { + forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5') + } + List precommitTasks = [ configureCheckstyle(project), configureForbiddenApisCli(project), @@ -39,7 +44,7 @@ class PrecommitTasks { project.tasks.create('licenseHeaders', LicenseHeadersTask.class), project.tasks.create('filepermissions', FilePermissionsTask.class), project.tasks.create('jarHell', JarHellTask.class), - project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) + configureThirdPartyAudit(project) ] // tasks with just tests don't need dependency licenses, so this flag makes adding @@ -75,32 +80,26 @@ class PrecommitTasks { return project.tasks.create(precommitOptions) } - private static Task configureForbiddenApisCli(Project project) { - Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar") - project.dependencies { - forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5') + private static Task configureThirdPartyAudit(Project project) { + ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) + ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') + thirdPartyAuditTask.configure { + dependsOn(buildResources) + signatureFile = buildResources.copy("forbidden/third-party-audit.txt") + javaHome = project.runtimeJavaHome } - Task forbiddenApisCli = project.tasks.create('forbiddenApis') + return thirdPartyAuditTask + } + private static Task configureForbiddenApisCli(Project project) { + Task forbiddenApisCli = project.tasks.create('forbiddenApis') project.sourceSets.forEach { sourceSet -> forbiddenApisCli.dependsOn( project.tasks.create(sourceSet.getTaskName('forbiddenApis', null), ForbiddenApisCliTask) { ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') dependsOn(buildResources) - execAction = { spec -> - spec.classpath = project.files( - project.configurations.forbiddenApisCliJar, - sourceSet.compileClasspath, - sourceSet.runtimeClasspath - ) - spec.executable = "${project.runtimeJavaHome}/bin/java" - } - inputs.files( - forbiddenApisConfiguration, - sourceSet.compileClasspath, - sourceSet.runtimeClasspath - ) - + it.sourceSet = sourceSet + javaHome = project.runtimeJavaHome targetCompatibility = project.compilerJavaVersion bundledSignatures = [ "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out" diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy deleted file mode 100644 index 52b13a56644..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.precommit; - -import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin -import org.apache.tools.ant.BuildEvent; -import org.apache.tools.ant.BuildException; -import org.apache.tools.ant.BuildListener; -import org.apache.tools.ant.BuildLogger; -import org.apache.tools.ant.DefaultLogger; -import org.apache.tools.ant.Project; -import org.elasticsearch.gradle.AntTask; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.file.FileCollection; -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.OutputFile - -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Basic static checking to keep tabs on third party JARs - */ -public class ThirdPartyAuditTask extends AntTask { - - // patterns for classes to exclude, because we understand their issues - private List excludes = []; - - /** - * Input for the task. Set javadoc for {#link getJars} for more. Protected - * so the afterEvaluate closure in the constructor can write it. - */ - protected FileCollection jars; - - /** - * Classpath against which to run the third patty audit. Protected so the - * afterEvaluate closure in the constructor can write it. - */ - protected FileCollection classpath; - - /** - * We use a simple "marker" file that we touch when the task succeeds - * as the task output. This is compared against the modified time of the - * inputs (ie the jars/class files). - */ - @OutputFile - File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit') - - ThirdPartyAuditTask() { - // we depend on this because its the only reliable configuration - // this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price. - dependsOn(project.configurations.testCompile); - description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"; - - project.afterEvaluate { - Configuration configuration = project.configurations.findByName('runtime') - Configuration compileOnly = project.configurations.findByName('compileOnly') - if (configuration == null) { - // some projects apparently do not have 'runtime'? what a nice inconsistency, - // basically only serves to waste time in build logic! - configuration = project.configurations.findByName('testCompile') - } - assert configuration != null - if (project.plugins.hasPlugin(ShadowPlugin)) { - Configuration original = configuration - configuration = project.configurations.create('thirdPartyAudit') - configuration.extendsFrom(original, project.configurations.bundle) - } - if (compileOnly == null) { - classpath = configuration - } else { - classpath = project.files(configuration, compileOnly) - } - - // we only want third party dependencies. - jars = configuration.fileCollection({ dependency -> - dependency.group.startsWith("org.elasticsearch") == false - }); - - // we don't want provided dependencies, which we have already scanned. e.g. don't - // scan ES core's dependencies for every single plugin - if (compileOnly != null) { - jars -= compileOnly - } - inputs.files(jars) - onlyIf { jars.isEmpty() == false } - } - } - - /** - * classes that should be excluded from the scan, - * e.g. because we know what sheisty stuff those particular classes are up to. - */ - public void setExcludes(String[] classes) { - for (String s : classes) { - if (s.indexOf('*') != -1) { - throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!"); - } - } - excludes = classes.sort(); - } - - /** - * Returns current list of exclusions. - */ - @Input - public List getExcludes() { - return excludes; - } - - // yes, we parse Uwe Schindler's errors to find missing classes, and to keep a continuous audit. Just don't let him know! - static final Pattern MISSING_CLASS_PATTERN = - Pattern.compile(/WARNING: The referenced class '(.*)' cannot be loaded\. Please fix the classpath\!/); - - static final Pattern VIOLATION_PATTERN = - Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/); - - // we log everything and capture errors and handle them with our whitelist - // this is important, as we detect stale whitelist entries, workaround forbidden apis bugs, - // and it also allows whitelisting missing classes! - static class EvilLogger extends DefaultLogger { - final Set missingClasses = new TreeSet<>(); - final Map> violations = new TreeMap<>(); - String previousLine = null; - - @Override - public void messageLogged(BuildEvent event) { - if (event.getTask().getClass() == de.thetaphi.forbiddenapis.ant.AntTask.class) { - if (event.getPriority() == Project.MSG_WARN) { - Matcher m = MISSING_CLASS_PATTERN.matcher(event.getMessage()); - if (m.matches()) { - missingClasses.add(m.group(1).replace('.', '/') + ".class"); - } - - // Reset the priority of the event to DEBUG, so it doesn't - // pollute the build output - event.setMessage(event.getMessage(), Project.MSG_DEBUG); - } else if (event.getPriority() == Project.MSG_ERR) { - Matcher m = VIOLATION_PATTERN.matcher(event.getMessage()); - if (m.matches()) { - String violation = previousLine + '\n' + event.getMessage(); - String clazz = m.group(1).replace('.', '/') + ".class"; - List current = violations.get(clazz); - if (current == null) { - current = new ArrayList<>(); - violations.put(clazz, current); - } - current.add(violation); - } - previousLine = event.getMessage(); - } - } - super.messageLogged(event); - } - } - - @Override - protected BuildLogger makeLogger(PrintStream stream, int outputLevel) { - DefaultLogger log = new EvilLogger(); - log.errorPrintStream = stream; - log.outputPrintStream = stream; - log.messageOutputLevel = outputLevel; - return log; - } - - @Override - protected void runAnt(AntBuilder ant) { - ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask); - - // print which jars we are going to scan, always - // this is not the time to try to be succinct! Forbidden will print plenty on its own! - Set names = new TreeSet<>(); - for (File jar : jars) { - names.add(jar.getName()); - } - - // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first, - // and then remove our temp dir afterwards. don't complain: try it yourself. - // we don't use gradle temp dir handling, just google it, or try it yourself. - - File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit'); - - // clean up any previous mess (if we failed), then unzip everything to one directory - ant.delete(dir: tmpDir.getAbsolutePath()); - tmpDir.mkdirs(); - for (File jar : jars) { - ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath()); - } - - // convert exclusion class names to binary file names - List excludedFiles = excludes.collect {it.replace('.', '/') + ".class"} - Set excludedSet = new TreeSet<>(excludedFiles); - - // jarHellReprise - Set sheistySet = getSheistyClasses(tmpDir.toPath()); - - try { - ant.thirdPartyAudit(failOnUnsupportedJava: false, - failOnMissingClasses: false, - classpath: classpath.asPath) { - fileset(dir: tmpDir) - signatures { - string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8')) - } - } - } catch (BuildException ignore) {} - - EvilLogger evilLogger = null; - for (BuildListener listener : ant.project.getBuildListeners()) { - if (listener instanceof EvilLogger) { - evilLogger = (EvilLogger) listener; - break; - } - } - assert evilLogger != null; - - // keep our whitelist up to date - Set bogusExclusions = new TreeSet<>(excludedSet); - bogusExclusions.removeAll(sheistySet); - bogusExclusions.removeAll(evilLogger.missingClasses); - bogusExclusions.removeAll(evilLogger.violations.keySet()); - if (!bogusExclusions.isEmpty()) { - throw new IllegalStateException("Invalid exclusions, nothing is wrong with these classes: " + bogusExclusions); - } - - // don't duplicate classes with the JDK - sheistySet.removeAll(excludedSet); - if (!sheistySet.isEmpty()) { - throw new IllegalStateException("JAR HELL WITH JDK! " + sheistySet); - } - - // don't allow a broken classpath - evilLogger.missingClasses.removeAll(excludedSet); - if (!evilLogger.missingClasses.isEmpty()) { - throw new IllegalStateException("CLASSES ARE MISSING! " + evilLogger.missingClasses); - } - - // don't use internal classes - evilLogger.violations.keySet().removeAll(excludedSet); - if (!evilLogger.violations.isEmpty()) { - throw new IllegalStateException("VIOLATIONS WERE FOUND! " + evilLogger.violations); - } - - // clean up our mess (if we succeed) - ant.delete(dir: tmpDir.getAbsolutePath()); - - successMarker.setText("", 'UTF-8') - } - - /** - * check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk! - */ - private Set getSheistyClasses(Path root) { - // system.parent = extensions loader. - // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!). - // but groovy/gradle needs to work at all first! - ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); - assert ext != null; - - Set sheistySet = new TreeSet<>(); - Files.walkFileTree(root, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - String entry = root.relativize(file).toString().replace('\\', '/'); - if (entry.endsWith(".class")) { - if (ext.getResource(entry) != null) { - sheistySet.add(entry); - } - } - return FileVisitResult.CONTINUE; - } - }); - return sheistySet; - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index a2484e9c5fc..a5d3b41339d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -53,6 +53,8 @@ public class StandaloneRestTestPlugin implements Plugin { // only setup tests to build project.sourceSets.create('test') + // create a compileOnly configuration as others might expect it + project.configurations.create("compileOnly") project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}") project.eclipse.classpath.sourceSets = [project.sourceSets.test] diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java new file mode 100644 index 00000000000..60de1981f98 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class JdkJarHellCheck { + + private Set detected = new HashSet<>(); + + private void scanForJDKJarHell(Path root) throws IOException { + // system.parent = extensions loader. + // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!) + ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); + assert ext != null; + + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String entry = root.relativize(file).toString().replace('\\', '/'); + if (entry.endsWith(".class")) { + if (ext.getResource(entry) != null) { + detected.add( + entry + .replace("/", ".") + .replace(".class","") + ); + } + } + return FileVisitResult.CONTINUE; + } + }); + } + + public Set getDetected() { + return Collections.unmodifiableSet(detected); + } + + public static void main(String[] argv) throws IOException { + JdkJarHellCheck checker = new JdkJarHellCheck(); + for (String location : argv) { + Path path = Paths.get(location); + if (Files.exists(path) == false) { + throw new IllegalArgumentException("Path does not exist: " + path); + } + checker.scanForJDKJarHell(path); + } + if (checker.getDetected().isEmpty()) { + System.exit(0); + } else { + checker.getDetected().forEach(System.out::println); + System.exit(1); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java index 21a0597b38a..46e5d84a2f2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java @@ -18,10 +18,9 @@ */ package org.elasticsearch.gradle.precommit; -import de.thetaphi.forbiddenapis.cli.CliMain; -import org.gradle.api.Action; import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; +import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; @@ -29,6 +28,7 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.TaskAction; import org.gradle.process.JavaExecSpec; @@ -50,7 +50,8 @@ public class ForbiddenApisCliTask extends DefaultTask { private Set suppressAnnotations = new LinkedHashSet<>(); private JavaVersion targetCompatibility; private FileCollection classesDirs; - private Action execAction; + private SourceSet sourceSet; + private String javaHome; @Input public JavaVersion getTargetCompatibility() { @@ -69,14 +70,6 @@ public class ForbiddenApisCliTask extends DefaultTask { } } - public Action getExecAction() { - return execAction; - } - - public void setExecAction(Action execAction) { - this.execAction = execAction; - } - @OutputFile public File getMarkerFile() { return new File( @@ -131,11 +124,41 @@ public class ForbiddenApisCliTask extends DefaultTask { this.suppressAnnotations = suppressAnnotations; } + @InputFiles + public FileCollection getClassPathFromSourceSet() { + return getProject().files( + sourceSet.getCompileClasspath(), + sourceSet.getRuntimeClasspath() + ); + } + + public void setSourceSet(SourceSet sourceSet) { + this.sourceSet = sourceSet; + } + + @InputFiles + public Configuration getForbiddenAPIsConfiguration() { + return getProject().getConfigurations().getByName("forbiddenApisCliJar"); + } + + @Input + public String getJavaHome() { + return javaHome; + } + + public void setJavaHome(String javaHome) { + this.javaHome = javaHome; + } + @TaskAction public void runForbiddenApisAndWriteMarker() throws IOException { getProject().javaexec((JavaExecSpec spec) -> { - execAction.execute(spec); - spec.setMain(CliMain.class.getName()); + spec.classpath( + getForbiddenAPIsConfiguration(), + getClassPathFromSourceSet() + ); + spec.setExecutable(getJavaHome() + "/bin/java"); + spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain"); // build the command line getSignaturesFiles().forEach(file -> spec.args("-f", file.getAbsolutePath())); getSuppressAnnotations().forEach(annotation -> spec.args("--suppressannotation", annotation)); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java new file mode 100644 index 00000000000..d1939d5c652 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -0,0 +1,288 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit; + +import org.apache.commons.io.output.NullOutputStream; +import org.elasticsearch.gradle.JdkJarHellCheck; +import org.elasticsearch.test.NamingConventionsCheck; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.StopExecutionException; +import org.gradle.api.tasks.TaskAction; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class ThirdPartyAuditTask extends DefaultTask { + + private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile( + "WARNING: The referenced class '(.*)' cannot be loaded\\. Please fix the classpath!" + ); + + private static final Pattern VIOLATION_PATTERN = Pattern.compile( + "\\s\\sin ([a-zA-Z0-9$.]+) \\(.*\\)" + ); + + /** + * patterns for classes to exclude, because we understand their issues + */ + private Set excludes = new TreeSet<>(); + + private File signatureFile; + + private String javaHome; + + @InputFiles + public Configuration getForbiddenAPIsConfiguration() { + return getProject().getConfigurations().getByName("forbiddenApisCliJar"); + } + + @InputFile + public File getSignatureFile() { + return signatureFile; + } + + public void setSignatureFile(File signatureFile) { + this.signatureFile = signatureFile; + } + + @InputFiles + public Configuration getRuntimeConfiguration() { + Configuration runtime = getProject().getConfigurations().findByName("runtime"); + if (runtime == null) { + return getProject().getConfigurations().getByName("testCompile"); + } + return runtime; + } + + @Input + public String getJavaHome() { + return javaHome; + } + + public void setJavaHome(String javaHome) { + this.javaHome = javaHome; + } + + @InputFiles + public Configuration getCompileOnlyConfiguration() { + return getProject().getConfigurations().getByName("compileOnly"); + } + + @OutputDirectory + public File getJarExpandDir() { + return new File( + new File(getProject().getBuildDir(), "precommit/thirdPartyAudit"), + getName() + ); + } + + public void setExcludes(String... classes) { + excludes.clear(); + for (String each : classes) { + if (each.indexOf('*') != -1) { + throw new IllegalArgumentException("illegal third party audit exclusion: '" + each + "', wildcards are not permitted!"); + } + excludes.add(each); + } + } + + @Input + public Set getExcludes() { + return Collections.unmodifiableSet(excludes); + } + + @TaskAction + public void runThirdPartyAudit() throws IOException { + FileCollection jars = getJarsToScan(); + + extractJars(jars); + + final String forbiddenApisOutput = runForbiddenAPIsCli(); + + final Set missingClasses = new TreeSet<>(); + Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput); + while (missingMatcher.find()) { + missingClasses.add(missingMatcher.group(1)); + } + + final Set violationsClasses = new TreeSet<>(); + Matcher violationMatcher = VIOLATION_PATTERN.matcher(forbiddenApisOutput); + while (violationMatcher.find()) { + violationsClasses.add(violationMatcher.group(1)); + } + + Set jdkJarHellClasses = runJdkJarHellCheck(); + + assertNoPointlessExclusions(missingClasses, violationsClasses, jdkJarHellClasses); + + assertNoMissingAndViolations(missingClasses, violationsClasses); + + assertNoJarHell(jdkJarHellClasses); + } + + private void extractJars(FileCollection jars) { + File jarExpandDir = getJarExpandDir(); + jars.forEach(jar -> + getProject().copy(spec -> { + spec.from(getProject().zipTree(jar)); + spec.into(jarExpandDir); + }) + ); + } + + private void assertNoJarHell(Set jdkJarHellClasses) { + jdkJarHellClasses.removeAll(excludes); + if (jdkJarHellClasses.isEmpty() == false) { + throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(jdkJarHellClasses)); + } + } + + private void assertNoMissingAndViolations(Set missingClasses, Set violationsClasses) { + missingClasses.removeAll(excludes); + violationsClasses.removeAll(excludes); + String missingText = formatClassList(missingClasses); + String violationsText = formatClassList(violationsClasses); + if (missingText.isEmpty() && violationsText.isEmpty()) { + getLogger().info("Third party audit passed successfully"); + } else { + throw new IllegalStateException( + "Audit of third party dependencies failed:\n" + + (missingText.isEmpty() ? "" : "Missing classes:\n" + missingText) + + (violationsText.isEmpty() ? "" : "Classes with violations:\n" + violationsText) + ); + } + } + + private void assertNoPointlessExclusions(Set missingClasses, Set violationsClasses, Set jdkJarHellClasses) { + // keep our whitelist up to date + Set bogusExclusions = new TreeSet<>(excludes); + bogusExclusions.removeAll(missingClasses); + bogusExclusions.removeAll(jdkJarHellClasses); + bogusExclusions.removeAll(violationsClasses); + if (bogusExclusions.isEmpty() == false) { + throw new IllegalStateException( + "Invalid exclusions, nothing is wrong with these classes: " + formatClassList(bogusExclusions) + ); + } + } + + private String runForbiddenAPIsCli() throws IOException { + ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); + getProject().javaexec(spec -> { + spec.setExecutable(javaHome + "/bin/java"); + spec.classpath( + getForbiddenAPIsConfiguration(), + getRuntimeConfiguration(), + getCompileOnlyConfiguration() + ); + spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain"); + spec.args( + "-f", getSignatureFile().getAbsolutePath(), + "-d", getJarExpandDir(), + "--allowmissingclasses" + ); + spec.setErrorOutput(errorOut); + if (getLogger().isInfoEnabled() == false) { + spec.setStandardOutput(new NullOutputStream()); + } + spec.setIgnoreExitValue(true); + }); + final String forbiddenApisOutput; + try (ByteArrayOutputStream outputStream = errorOut) { + forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name()); + } + if (getLogger().isInfoEnabled()) { + getLogger().info(forbiddenApisOutput); + } + return forbiddenApisOutput; + } + + private FileCollection getJarsToScan() { + FileCollection jars = getRuntimeConfiguration() + .fileCollection(dep -> dep.getGroup().startsWith("org.elasticsearch") == false); + Configuration compileOnlyConfiguration = getCompileOnlyConfiguration(); + // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin + if (compileOnlyConfiguration != null) { + jars.minus(compileOnlyConfiguration); + } + if (jars.isEmpty()) { + throw new StopExecutionException("No jars to scan"); + } + return jars; + } + + private String formatClassList(Set classList) { + return classList.stream() + .map(name -> " * " + name) + .collect(Collectors.joining("\n")); + } + + private Set runJdkJarHellCheck() throws IOException { + ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); + ExecResult execResult = getProject().javaexec(spec -> { + URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation(); + if (location.getProtocol().equals("file") == false) { + throw new GradleException("Unexpected location for NamingConventionCheck class: " + location); + } + try { + spec.classpath( + location.toURI().getPath(), + getRuntimeConfiguration(), + getCompileOnlyConfiguration() + ); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + spec.setMain(JdkJarHellCheck.class.getName()); + spec.args(getJarExpandDir()); + spec.setIgnoreExitValue(true); + spec.setExecutable(javaHome + "/bin/java"); + spec.setStandardOutput(standardOut); + }); + if (execResult.getExitValue() == 0) { + return Collections.emptySet(); + } + final String jdkJarHellCheckList; + try (ByteArrayOutputStream outputStream = standardOut) { + jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8.name()); + } + return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n"))); + } + + +} diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index 6f177f7b7f5..3dae3d3642c 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -128,7 +128,7 @@ thirdPartyAudit.excludes = [ ] // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9) -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.Binder', 'javax.xml.bind.ContextFinder$1', diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index b1c3b62fd6e..e32ba6948d6 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -87,7 +87,7 @@ thirdPartyAudit.excludes = [ 'org.apache.log.Logger', ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.DatatypeConverter', 'javax.xml.bind.JAXBContext' diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 1a6aa809de0..6cd55f682c8 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -2106,7 +2106,27 @@ thirdPartyAudit.excludes = [ 'ucar.nc2.dataset.NetcdfDataset' ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + // TODO: Why is this needed ? + 'com.sun.javadoc.ClassDoc', + 'com.sun.javadoc.Doc', + 'com.sun.javadoc.Doclet', + 'com.sun.javadoc.ExecutableMemberDoc', + 'com.sun.javadoc.FieldDoc', + 'com.sun.javadoc.MethodDoc', + 'com.sun.javadoc.PackageDoc', + 'com.sun.javadoc.Parameter', + 'com.sun.javadoc.ProgramElementDoc', + 'com.sun.javadoc.RootDoc', + 'com.sun.javadoc.SourcePosition', + 'com.sun.javadoc.Tag', + 'com.sun.javadoc.Type', + 'com.sun.tools.javadoc.Main' + ] +} + +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.activation.ActivationDataFlavor', 'javax.activation.CommandMap', diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 6debaf5282f..557dcaa5fae 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -582,6 +582,25 @@ thirdPartyAudit.excludes = [ 'com.squareup.okhttp.ResponseBody' ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += ['javax.xml.bind.annotation.adapters.HexBinaryAdapter'] } + +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + // TODO: Why is this needed ? + 'com.sun.javadoc.AnnotationDesc', + 'com.sun.javadoc.AnnotationTypeDoc', + 'com.sun.javadoc.ClassDoc', + 'com.sun.javadoc.ConstructorDoc', + 'com.sun.javadoc.Doc', + 'com.sun.javadoc.DocErrorReporter', + 'com.sun.javadoc.FieldDoc', + 'com.sun.javadoc.LanguageVersion', + 'com.sun.javadoc.MethodDoc', + 'com.sun.javadoc.PackageDoc', + 'com.sun.javadoc.ProgramElementDoc', + 'com.sun.javadoc.RootDoc', + 'com.sun.tools.doclets.standard.Standard' + ] +} diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 7f0ca209db7..5d248b22caf 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -447,7 +447,7 @@ thirdPartyAudit.excludes = [ ] // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9) -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.Binder', 'javax.xml.bind.ContextFinder$1', diff --git a/server/build.gradle b/server/build.gradle index b22a93a702c..f8a604941f7 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -304,17 +304,22 @@ thirdPartyAudit.excludes = [ 'com.google.common.geometry.S2LatLng', ] -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { - // Used by Log4J 2.11.1 +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ - 'java.io.ObjectInputFilter', - 'java.io.ObjectInputFilter$Config', - 'java.io.ObjectInputFilter$FilterInfo', - 'java.io.ObjectInputFilter$Status' + // Used by Log4J 2.11.1 + 'java.io.ObjectInputFilter', + 'java.io.ObjectInputFilter$Config', + 'java.io.ObjectInputFilter$FilterInfo', + 'java.io.ObjectInputFilter$Status', + // added in 9 + 'java.lang.ProcessHandle', + 'java.lang.StackWalker', + 'java.lang.StackWalker$Option', + 'java.lang.StackWalker$StackFrame' ] } -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter'] } diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle index 0f02283e537..2da90656414 100644 --- a/test/logger-usage/build.gradle +++ b/test/logger-usage/build.gradle @@ -44,7 +44,7 @@ thirdPartyAudit.excludes = [ 'org.osgi.framework.wiring.BundleWiring' ] -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { // Used by Log4J 2.11.1 thirdPartyAudit.excludes += [ 'java.io.ObjectInputFilter', @@ -52,4 +52,13 @@ if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { 'java.io.ObjectInputFilter$FilterInfo', 'java.io.ObjectInputFilter$Status' ] +} + +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + 'java.lang.ProcessHandle', + 'java.lang.StackWalker', + 'java.lang.StackWalker$Option', + 'java.lang.StackWalker$StackFrame' + ] } \ No newline at end of file diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index f2c78e12258..71b22531cca 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -242,7 +242,7 @@ thirdPartyAudit.excludes = [ 'javax.persistence.EntityManagerFactory', 'javax.persistence.EntityTransaction', 'javax.persistence.LockModeType', - 'javax/persistence/Query', + 'javax.persistence.Query', // [missing classes] OpenSAML storage and HttpClient cache have optional memcache support 'net.spy.memcached.CASResponse', 'net.spy.memcached.CASValue', @@ -266,7 +266,7 @@ thirdPartyAudit.excludes = [ 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', ] -if (JavaVersion.current() > JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'javax.xml.bind.JAXBContext', 'javax.xml.bind.JAXBElement', diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle index 345318d20b8..ee99c36b906 100644 --- a/x-pack/plugin/sql/sql-action/build.gradle +++ b/x-pack/plugin/sql/sql-action/build.gradle @@ -140,7 +140,7 @@ thirdPartyAudit.excludes = [ 'org.zeromq.ZMQ' ] -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { // Used by Log4J 2.11.1 thirdPartyAudit.excludes += [ 'java.io.ObjectInputFilter', @@ -148,4 +148,13 @@ if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { 'java.io.ObjectInputFilter$FilterInfo', 'java.io.ObjectInputFilter$Status' ] +} + +if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) { + thirdPartyAudit.excludes += [ + 'java.lang.ProcessHandle', + 'java.lang.StackWalker', + 'java.lang.StackWalker$Option', + 'java.lang.StackWalker$StackFrame' + ] } \ No newline at end of file diff --git a/x-pack/plugin/watcher/build.gradle b/x-pack/plugin/watcher/build.gradle index 3a9d759c46d..3412cafc4f4 100644 --- a/x-pack/plugin/watcher/build.gradle +++ b/x-pack/plugin/watcher/build.gradle @@ -68,7 +68,7 @@ thirdPartyAudit.excludes = [ ] // pulled in as external dependency to work on java 9 -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { +if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'com.sun.activation.registries.MailcapParseException', 'javax.activation.ActivationDataFlavor',