honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Run Third party audit with forbidden APIs CLI (part3/3) (#33052) 

The new implementation is functional equivalent with the old, ant based one.
It parses task standard error to get the missing classes and violations in the same way.
I considered re-using ForbiddenApisCliTask but Gradle makes it hard to build inheritance with tasks that have task actions , since the order of the task actions can't be controlled.
This inheritance isn't dully desired either as the third party audit task is much more opinionated and we don't want to expose some of the configuration.
We could probably extract a common base class without any task actions, but probably more trouble than it's worth.

Closes #31715
This commit is contained in:
Alpar Torok 2018-08-28 10:03:30 +03:00 committed by GitHub
parent 71d5c66fd3
commit 2cc611604f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 505 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
buildSrc/build.gradle
View File

@ -102,7 +102,6 @@ dependencies {
compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
compile 'de.thetaphi:forbiddenapis:2.5'
compile 'org.apache.rat:apache-rat:0.11'
compile "org.elasticsearch:jna:4.5.1"
compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'

39
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View File

@ -31,6 +31,11 @@ class PrecommitTasks {
/** Adds a precommit task, which depends on non-test verification tasks. */
public static Task create(Project project, boolean includeDependencyLicenses) {
Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar")
project.dependencies {
forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5')
}
List<Task> precommitTasks = [
configureCheckstyle(project),
configureForbiddenApisCli(project),
@ -39,7 +44,7 @@ class PrecommitTasks {
project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
project.tasks.create('filepermissions', FilePermissionsTask.class),
project.tasks.create('jarHell', JarHellTask.class),
project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
configureThirdPartyAudit(project)
]
// tasks with just tests don't need dependency licenses, so this flag makes adding
@ -75,32 +80,26 @@ class PrecommitTasks {
return project.tasks.create(precommitOptions)
}
private static Task configureForbiddenApisCli(Project project) {
Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar")
project.dependencies {
forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5')
private static Task configureThirdPartyAudit(Project project) {
ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
thirdPartyAuditTask.configure {
dependsOn(buildResources)
signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
javaHome = project.runtimeJavaHome
}
Task forbiddenApisCli = project.tasks.create('forbiddenApis')
return thirdPartyAuditTask
}
private static Task configureForbiddenApisCli(Project project) {
Task forbiddenApisCli = project.tasks.create('forbiddenApis')
project.sourceSets.forEach { sourceSet ->
forbiddenApisCli.dependsOn(
project.tasks.create(sourceSet.getTaskName('forbiddenApis', null), ForbiddenApisCliTask) {
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
dependsOn(buildResources)
execAction = { spec ->
spec.classpath = project.files(
project.configurations.forbiddenApisCliJar,
sourceSet.compileClasspath,
sourceSet.runtimeClasspath
)
spec.executable = "${project.runtimeJavaHome}/bin/java"
}
inputs.files(
forbiddenApisConfiguration,
sourceSet.compileClasspath,
sourceSet.runtimeClasspath
)
it.sourceSet = sourceSet
javaHome = project.runtimeJavaHome
targetCompatibility = project.compilerJavaVersion
bundledSignatures = [
"jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"

297
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
View File

@ -1,297 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit;
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
import org.apache.tools.ant.BuildEvent;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.BuildListener;
import org.apache.tools.ant.BuildLogger;
import org.apache.tools.ant.DefaultLogger;
import org.apache.tools.ant.Project;
import org.elasticsearch.gradle.AntTask;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Basic static checking to keep tabs on third party JARs
*/
public class ThirdPartyAuditTask extends AntTask {
// patterns for classes to exclude, because we understand their issues
private List<String> excludes = [];
/**
* Input for the task. Set javadoc for {#link getJars} for more. Protected
* so the afterEvaluate closure in the constructor can write it.
*/
protected FileCollection jars;
/**
* Classpath against which to run the third patty audit. Protected so the
* afterEvaluate closure in the constructor can write it.
*/
protected FileCollection classpath;
/**
* We use a simple "marker" file that we touch when the task succeeds
* as the task output. This is compared against the modified time of the
* inputs (ie the jars/class files).
*/
@OutputFile
File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit')
ThirdPartyAuditTask() {
// we depend on this because its the only reliable configuration
// this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price.
dependsOn(project.configurations.testCompile);
description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'";
project.afterEvaluate {
Configuration configuration = project.configurations.findByName('runtime')
Configuration compileOnly = project.configurations.findByName('compileOnly')
if (configuration == null) {
// some projects apparently do not have 'runtime'? what a nice inconsistency,
// basically only serves to waste time in build logic!
configuration = project.configurations.findByName('testCompile')
}
assert configuration != null
if (project.plugins.hasPlugin(ShadowPlugin)) {
Configuration original = configuration
configuration = project.configurations.create('thirdPartyAudit')
configuration.extendsFrom(original, project.configurations.bundle)
}
if (compileOnly == null) {
classpath = configuration
} else {
classpath = project.files(configuration, compileOnly)
}
// we only want third party dependencies.
jars = configuration.fileCollection({ dependency ->
dependency.group.startsWith("org.elasticsearch") == false
});
// we don't want provided dependencies, which we have already scanned. e.g. don't
// scan ES core's dependencies for every single plugin
if (compileOnly != null) {
jars -= compileOnly
}
inputs.files(jars)
onlyIf { jars.isEmpty() == false }
}
}
/**
* classes that should be excluded from the scan,
* e.g. because we know what sheisty stuff those particular classes are up to.
*/
public void setExcludes(String[] classes) {
for (String s : classes) {
if (s.indexOf('*') != -1) {
throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!");
}
}
excludes = classes.sort();
}
/**
* Returns current list of exclusions.
*/
@Input
public List<String> getExcludes() {
return excludes;
}
// yes, we parse Uwe Schindler's errors to find missing classes, and to keep a continuous audit. Just don't let him know!
static final Pattern MISSING_CLASS_PATTERN =
Pattern.compile(/WARNING: The referenced class '(.*)' cannot be loaded\. Please fix the classpath\!/);
static final Pattern VIOLATION_PATTERN =
Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/);
// we log everything and capture errors and handle them with our whitelist
// this is important, as we detect stale whitelist entries, workaround forbidden apis bugs,
// and it also allows whitelisting missing classes!
static class EvilLogger extends DefaultLogger {
final Set<String> missingClasses = new TreeSet<>();
final Map<String,List<String>> violations = new TreeMap<>();
String previousLine = null;
@Override
public void messageLogged(BuildEvent event) {
if (event.getTask().getClass() == de.thetaphi.forbiddenapis.ant.AntTask.class) {
if (event.getPriority() == Project.MSG_WARN) {
Matcher m = MISSING_CLASS_PATTERN.matcher(event.getMessage());
if (m.matches()) {
missingClasses.add(m.group(1).replace('.', '/') + ".class");
}
// Reset the priority of the event to DEBUG, so it doesn't
// pollute the build output
event.setMessage(event.getMessage(), Project.MSG_DEBUG);
} else if (event.getPriority() == Project.MSG_ERR) {
Matcher m = VIOLATION_PATTERN.matcher(event.getMessage());
if (m.matches()) {
String violation = previousLine + '\n' + event.getMessage();
String clazz = m.group(1).replace('.', '/') + ".class";
List<String> current = violations.get(clazz);
if (current == null) {
current = new ArrayList<>();
violations.put(clazz, current);
}
current.add(violation);
}
previousLine = event.getMessage();
}
}
super.messageLogged(event);
}
}
@Override
protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
DefaultLogger log = new EvilLogger();
log.errorPrintStream = stream;
log.outputPrintStream = stream;
log.messageOutputLevel = outputLevel;
return log;
}
@Override
protected void runAnt(AntBuilder ant) {
ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask);
// print which jars we are going to scan, always
// this is not the time to try to be succinct! Forbidden will print plenty on its own!
Set<String> names = new TreeSet<>();
for (File jar : jars) {
names.add(jar.getName());
}
// TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,
// and then remove our temp dir afterwards. don't complain: try it yourself.
// we don't use gradle temp dir handling, just google it, or try it yourself.
File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit');
// clean up any previous mess (if we failed), then unzip everything to one directory
ant.delete(dir: tmpDir.getAbsolutePath());
tmpDir.mkdirs();
for (File jar : jars) {
ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath());
}
// convert exclusion class names to binary file names
List<String> excludedFiles = excludes.collect {it.replace('.', '/') + ".class"}
Set<String> excludedSet = new TreeSet<>(excludedFiles);
// jarHellReprise
Set<String> sheistySet = getSheistyClasses(tmpDir.toPath());
try {
ant.thirdPartyAudit(failOnUnsupportedJava: false,
failOnMissingClasses: false,
classpath: classpath.asPath) {
fileset(dir: tmpDir)
signatures {
string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8'))
}
}
} catch (BuildException ignore) {}
EvilLogger evilLogger = null;
for (BuildListener listener : ant.project.getBuildListeners()) {
if (listener instanceof EvilLogger) {
evilLogger = (EvilLogger) listener;
break;
}
}
assert evilLogger != null;
// keep our whitelist up to date
Set<String> bogusExclusions = new TreeSet<>(excludedSet);
bogusExclusions.removeAll(sheistySet);
bogusExclusions.removeAll(evilLogger.missingClasses);
bogusExclusions.removeAll(evilLogger.violations.keySet());
if (!bogusExclusions.isEmpty()) {
throw new IllegalStateException("Invalid exclusions, nothing is wrong with these classes: " + bogusExclusions);
}
// don't duplicate classes with the JDK
sheistySet.removeAll(excludedSet);
if (!sheistySet.isEmpty()) {
throw new IllegalStateException("JAR HELL WITH JDK! " + sheistySet);
}
// don't allow a broken classpath
evilLogger.missingClasses.removeAll(excludedSet);
if (!evilLogger.missingClasses.isEmpty()) {
throw new IllegalStateException("CLASSES ARE MISSING! " + evilLogger.missingClasses);
}
// don't use internal classes
evilLogger.violations.keySet().removeAll(excludedSet);
if (!evilLogger.violations.isEmpty()) {
throw new IllegalStateException("VIOLATIONS WERE FOUND! " + evilLogger.violations);
}
// clean up our mess (if we succeed)
ant.delete(dir: tmpDir.getAbsolutePath());
successMarker.setText("", 'UTF-8')
}
/**
* check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk!
*/
private Set<String> getSheistyClasses(Path root) {
// system.parent = extensions loader.
// note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!).
// but groovy/gradle needs to work at all first!
ClassLoader ext = ClassLoader.getSystemClassLoader().getParent();
assert ext != null;
Set<String> sheistySet = new TreeSet<>();
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String entry = root.relativize(file).toString().replace('\\', '/');
if (entry.endsWith(".class")) {
if (ext.getResource(entry) != null) {
sheistySet.add(entry);
}
}
return FileVisitResult.CONTINUE;
}
});
return sheistySet;
}
}

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
View File

@ -53,6 +53,8 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
// only setup tests to build
project.sourceSets.create('test')
// create a compileOnly configuration as others might expect it
project.configurations.create("compileOnly")
project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
project.eclipse.classpath.sourceSets = [project.sourceSets.test]

81
buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
public class JdkJarHellCheck {
private Set<String> detected = new HashSet<>();
private void scanForJDKJarHell(Path root) throws IOException {
// system.parent = extensions loader.
// note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!)
ClassLoader ext = ClassLoader.getSystemClassLoader().getParent();
assert ext != null;
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
String entry = root.relativize(file).toString().replace('\\', '/');
if (entry.endsWith(".class")) {
if (ext.getResource(entry) != null) {
detected.add(
entry
.replace("/", ".")
.replace(".class","")
);
}
}
return FileVisitResult.CONTINUE;
}
});
}
public Set<String> getDetected() {
return Collections.unmodifiableSet(detected);
}
public static void main(String[] argv) throws IOException {
JdkJarHellCheck checker = new JdkJarHellCheck();
for (String location : argv) {
Path path = Paths.get(location);
if (Files.exists(path) == false) {
throw new IllegalArgumentException("Path does not exist: " + path);
}
checker.scanForJDKJarHell(path);
}
if (checker.getDetected().isEmpty()) {
System.exit(0);
} else {
checker.getDetected().forEach(System.out::println);
System.exit(1);
}
}
}

49
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java
View File

@ -18,10 +18,9 @@
*/
package org.elasticsearch.gradle.precommit;
import de.thetaphi.forbiddenapis.cli.CliMain;
import org.gradle.api.Action;
import org.gradle.api.DefaultTask;
import org.gradle.api.JavaVersion;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.FileCollection;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
@ -29,6 +28,7 @@ import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.SkipWhenEmpty;
import org.gradle.api.tasks.SourceSet;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.JavaExecSpec;
@ -50,7 +50,8 @@ public class ForbiddenApisCliTask extends DefaultTask {
private Set<String> suppressAnnotations = new LinkedHashSet<>();
private JavaVersion targetCompatibility;
private FileCollection classesDirs;
private Action<JavaExecSpec> execAction;
private SourceSet sourceSet;
private String javaHome;
@Input
public JavaVersion getTargetCompatibility() {
@ -69,14 +70,6 @@ public class ForbiddenApisCliTask extends DefaultTask {
}
}
public Action<JavaExecSpec> getExecAction() {
return execAction;
}
public void setExecAction(Action<JavaExecSpec> execAction) {
this.execAction = execAction;
}
@OutputFile
public File getMarkerFile() {
return new File(
@ -131,11 +124,41 @@ public class ForbiddenApisCliTask extends DefaultTask {
this.suppressAnnotations = suppressAnnotations;
}
@InputFiles
public FileCollection getClassPathFromSourceSet() {
return getProject().files(
sourceSet.getCompileClasspath(),
sourceSet.getRuntimeClasspath()
);
}
public void setSourceSet(SourceSet sourceSet) {
this.sourceSet = sourceSet;
}
@InputFiles
public Configuration getForbiddenAPIsConfiguration() {
return getProject().getConfigurations().getByName("forbiddenApisCliJar");
}
@Input
public String getJavaHome() {
return javaHome;
}
public void setJavaHome(String javaHome) {
this.javaHome = javaHome;
}
@TaskAction
public void runForbiddenApisAndWriteMarker() throws IOException {
getProject().javaexec((JavaExecSpec spec) -> {
execAction.execute(spec);
spec.setMain(CliMain.class.getName());
spec.classpath(
getForbiddenAPIsConfiguration(),
getClassPathFromSourceSet()
);
spec.setExecutable(getJavaHome() + "/bin/java");
spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain");
// build the command line
getSignaturesFiles().forEach(file -> spec.args("-f", file.getAbsolutePath()));
getSuppressAnnotations().forEach(annotation -> spec.args("--suppressannotation", annotation));

288
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java Normal file
View File

@ -0,0 +1,288 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit;
import org.apache.commons.io.output.NullOutputStream;
import org.elasticsearch.gradle.JdkJarHellCheck;
import org.elasticsearch.test.NamingConventionsCheck;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputFile;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.OutputDirectory;
import org.gradle.api.tasks.StopExecutionException;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.ExecResult;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class ThirdPartyAuditTask extends DefaultTask {
private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile(
"WARNING: The referenced class '(.*)' cannot be loaded\\. Please fix the classpath!"
);
private static final Pattern VIOLATION_PATTERN = Pattern.compile(
"\\s\\sin ([a-zA-Z0-9$.]+) \\(.*\\)"
);
/**
* patterns for classes to exclude, because we understand their issues
*/
private Set<String> excludes = new TreeSet<>();
private File signatureFile;
private String javaHome;
@InputFiles
public Configuration getForbiddenAPIsConfiguration() {
return getProject().getConfigurations().getByName("forbiddenApisCliJar");
}
@InputFile
public File getSignatureFile() {
return signatureFile;
}
public void setSignatureFile(File signatureFile) {
this.signatureFile = signatureFile;
}
@InputFiles
public Configuration getRuntimeConfiguration() {
Configuration runtime = getProject().getConfigurations().findByName("runtime");
if (runtime == null) {
return getProject().getConfigurations().getByName("testCompile");
}
return runtime;
}
@Input
public String getJavaHome() {
return javaHome;
}
public void setJavaHome(String javaHome) {
this.javaHome = javaHome;
}
@InputFiles
public Configuration getCompileOnlyConfiguration() {
return getProject().getConfigurations().getByName("compileOnly");
}
@OutputDirectory
public File getJarExpandDir() {
return new File(
new File(getProject().getBuildDir(), "precommit/thirdPartyAudit"),
getName()
);
}
public void setExcludes(String... classes) {
excludes.clear();
for (String each : classes) {
if (each.indexOf('*') != -1) {
throw new IllegalArgumentException("illegal third party audit exclusion: '" + each + "', wildcards are not permitted!");
}
excludes.add(each);
}
}
@Input
public Set<String> getExcludes() {
return Collections.unmodifiableSet(excludes);
}
@TaskAction
public void runThirdPartyAudit() throws IOException {
FileCollection jars = getJarsToScan();
extractJars(jars);
final String forbiddenApisOutput = runForbiddenAPIsCli();
final Set<String> missingClasses = new TreeSet<>();
Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput);
while (missingMatcher.find()) {
missingClasses.add(missingMatcher.group(1));
}
final Set<String> violationsClasses = new TreeSet<>();
Matcher violationMatcher = VIOLATION_PATTERN.matcher(forbiddenApisOutput);
while (violationMatcher.find()) {
violationsClasses.add(violationMatcher.group(1));
}
Set<String> jdkJarHellClasses = runJdkJarHellCheck();
assertNoPointlessExclusions(missingClasses, violationsClasses, jdkJarHellClasses);
assertNoMissingAndViolations(missingClasses, violationsClasses);
assertNoJarHell(jdkJarHellClasses);
}
private void extractJars(FileCollection jars) {
File jarExpandDir = getJarExpandDir();
jars.forEach(jar ->
getProject().copy(spec -> {
spec.from(getProject().zipTree(jar));
spec.into(jarExpandDir);
})
);
}
private void assertNoJarHell(Set<String> jdkJarHellClasses) {
jdkJarHellClasses.removeAll(excludes);
if (jdkJarHellClasses.isEmpty() == false) {
throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(jdkJarHellClasses));
}
}
private void assertNoMissingAndViolations(Set<String> missingClasses, Set<String> violationsClasses) {
missingClasses.removeAll(excludes);
violationsClasses.removeAll(excludes);
String missingText = formatClassList(missingClasses);
String violationsText = formatClassList(violationsClasses);
if (missingText.isEmpty() && violationsText.isEmpty()) {
getLogger().info("Third party audit passed successfully");
} else {
throw new IllegalStateException(
"Audit of third party dependencies failed:\n" +
(missingText.isEmpty() ? "" : "Missing classes:\n" + missingText) +
(violationsText.isEmpty() ? "" : "Classes with violations:\n" + violationsText)
);
}
}
private void assertNoPointlessExclusions(Set<String> missingClasses, Set<String> violationsClasses, Set<String> jdkJarHellClasses) {
// keep our whitelist up to date
Set<String> bogusExclusions = new TreeSet<>(excludes);
bogusExclusions.removeAll(missingClasses);
bogusExclusions.removeAll(jdkJarHellClasses);
bogusExclusions.removeAll(violationsClasses);
if (bogusExclusions.isEmpty() == false) {
throw new IllegalStateException(
"Invalid exclusions, nothing is wrong with these classes: " + formatClassList(bogusExclusions)
);
}
}
private String runForbiddenAPIsCli() throws IOException {
ByteArrayOutputStream errorOut = new ByteArrayOutputStream();
getProject().javaexec(spec -> {
spec.setExecutable(javaHome + "/bin/java");
spec.classpath(
getForbiddenAPIsConfiguration(),
getRuntimeConfiguration(),
getCompileOnlyConfiguration()
);
spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain");
spec.args(
"-f", getSignatureFile().getAbsolutePath(),
"-d", getJarExpandDir(),
"--allowmissingclasses"
);
spec.setErrorOutput(errorOut);
if (getLogger().isInfoEnabled() == false) {
spec.setStandardOutput(new NullOutputStream());
}
spec.setIgnoreExitValue(true);
});
final String forbiddenApisOutput;
try (ByteArrayOutputStream outputStream = errorOut) {
forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name());
}
if (getLogger().isInfoEnabled()) {
getLogger().info(forbiddenApisOutput);
}
return forbiddenApisOutput;
}
private FileCollection getJarsToScan() {
FileCollection jars = getRuntimeConfiguration()
.fileCollection(dep -> dep.getGroup().startsWith("org.elasticsearch") == false);
Configuration compileOnlyConfiguration = getCompileOnlyConfiguration();
// don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin
if (compileOnlyConfiguration != null) {
jars.minus(compileOnlyConfiguration);
}
if (jars.isEmpty()) {
throw new StopExecutionException("No jars to scan");
}
return jars;
}
private String formatClassList(Set<String> classList) {
return classList.stream()
.map(name -> " * " + name)
.collect(Collectors.joining("\n"));
}
private Set<String> runJdkJarHellCheck() throws IOException {
ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
ExecResult execResult = getProject().javaexec(spec -> {
URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();
if (location.getProtocol().equals("file") == false) {
throw new GradleException("Unexpected location for NamingConventionCheck class: " + location);
}
try {
spec.classpath(
location.toURI().getPath(),
getRuntimeConfiguration(),
getCompileOnlyConfiguration()
);
} catch (URISyntaxException e) {
throw new AssertionError(e);
}
spec.setMain(JdkJarHellCheck.class.getName());
spec.args(getJarExpandDir());
spec.setIgnoreExitValue(true);
spec.setExecutable(javaHome + "/bin/java");
spec.setStandardOutput(standardOut);
});
if (execResult.getExitValue() == 0) {
return Collections.emptySet();
}
final String jdkJarHellCheckList;
try (ByteArrayOutputStream outputStream = standardOut) {
jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8.name());
}
return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
}
}

2
plugins/discovery-azure-classic/build.gradle
View File

@ -128,7 +128,7 @@ thirdPartyAudit.excludes = [
]
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'javax.xml.bind.Binder',
'javax.xml.bind.ContextFinder$1',

2
plugins/discovery-ec2/build.gradle
View File

@ -87,7 +87,7 @@ thirdPartyAudit.excludes = [
'org.apache.log.Logger',
]
if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'javax.xml.bind.DatatypeConverter',
'javax.xml.bind.JAXBContext'

22
plugins/ingest-attachment/build.gradle
View File

@ -2106,7 +2106,27 @@ thirdPartyAudit.excludes = [
'ucar.nc2.dataset.NetcdfDataset'
]
if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
// TODO: Why is this needed ?
'com.sun.javadoc.ClassDoc',
'com.sun.javadoc.Doc',
'com.sun.javadoc.Doclet',
'com.sun.javadoc.ExecutableMemberDoc',
'com.sun.javadoc.FieldDoc',
'com.sun.javadoc.MethodDoc',
'com.sun.javadoc.PackageDoc',
'com.sun.javadoc.Parameter',
'com.sun.javadoc.ProgramElementDoc',
'com.sun.javadoc.RootDoc',
'com.sun.javadoc.SourcePosition',
'com.sun.javadoc.Tag',
'com.sun.javadoc.Type',
'com.sun.tools.javadoc.Main'
]
}
if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'javax.activation.ActivationDataFlavor',
'javax.activation.CommandMap',

21
plugins/repository-hdfs/build.gradle
View File

@ -582,6 +582,25 @@ thirdPartyAudit.excludes = [
'com.squareup.okhttp.ResponseBody'
]
if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += ['javax.xml.bind.annotation.adapters.HexBinaryAdapter']
}
if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
// TODO: Why is this needed ?
'com.sun.javadoc.AnnotationDesc',
'com.sun.javadoc.AnnotationTypeDoc',
'com.sun.javadoc.ClassDoc',
'com.sun.javadoc.ConstructorDoc',
'com.sun.javadoc.Doc',
'com.sun.javadoc.DocErrorReporter',
'com.sun.javadoc.FieldDoc',
'com.sun.javadoc.LanguageVersion',
'com.sun.javadoc.MethodDoc',
'com.sun.javadoc.PackageDoc',
'com.sun.javadoc.ProgramElementDoc',
'com.sun.javadoc.RootDoc',
'com.sun.tools.doclets.standard.Standard'
]
}

2
plugins/repository-s3/build.gradle
View File

@ -447,7 +447,7 @@ thirdPartyAudit.excludes = [
]
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'javax.xml.bind.Binder',
'javax.xml.bind.ContextFinder$1',

19
server/build.gradle
View File

@ -304,17 +304,22 @@ thirdPartyAudit.excludes = [
'com.google.common.geometry.S2LatLng',
]
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
// Used by Log4J 2.11.1
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'java.io.ObjectInputFilter',
'java.io.ObjectInputFilter$Config',
'java.io.ObjectInputFilter$FilterInfo',
'java.io.ObjectInputFilter$Status'
// Used by Log4J 2.11.1
'java.io.ObjectInputFilter',
'java.io.ObjectInputFilter$Config',
'java.io.ObjectInputFilter$FilterInfo',
'java.io.ObjectInputFilter$Status',
// added in 9
'java.lang.ProcessHandle',
'java.lang.StackWalker',
'java.lang.StackWalker$Option',
'java.lang.StackWalker$StackFrame'
]
}
if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
}

11
test/logger-usage/build.gradle
View File

@ -44,7 +44,7 @@ thirdPartyAudit.excludes = [
'org.osgi.framework.wiring.BundleWiring'
]
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
// Used by Log4J 2.11.1
thirdPartyAudit.excludes += [
'java.io.ObjectInputFilter',
@ -52,4 +52,13 @@ if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
'java.io.ObjectInputFilter$FilterInfo',
'java.io.ObjectInputFilter$Status'
]
}
if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'java.lang.ProcessHandle',
'java.lang.StackWalker',
'java.lang.StackWalker$Option',
'java.lang.StackWalker$StackFrame'
]
}

4
x-pack/plugin/security/build.gradle
View File

@ -242,7 +242,7 @@ thirdPartyAudit.excludes = [
'javax.persistence.EntityManagerFactory',
'javax.persistence.EntityTransaction',
'javax.persistence.LockModeType',
'javax/persistence/Query',
'javax.persistence.Query',
// [missing classes] OpenSAML storage and HttpClient cache have optional memcache support
'net.spy.memcached.CASResponse',
'net.spy.memcached.CASValue',
@ -266,7 +266,7 @@ thirdPartyAudit.excludes = [
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
]
if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'javax.xml.bind.JAXBContext',
'javax.xml.bind.JAXBElement',

11
x-pack/plugin/sql/sql-action/build.gradle
View File

@ -140,7 +140,7 @@ thirdPartyAudit.excludes = [
'org.zeromq.ZMQ'
]
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
// Used by Log4J 2.11.1
thirdPartyAudit.excludes += [
'java.io.ObjectInputFilter',
@ -148,4 +148,13 @@ if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
'java.io.ObjectInputFilter$FilterInfo',
'java.io.ObjectInputFilter$Status'
]
}
if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'java.lang.ProcessHandle',
'java.lang.StackWalker',
'java.lang.StackWalker$Option',
'java.lang.StackWalker$StackFrame'
]
}

2
x-pack/plugin/watcher/build.gradle
View File

@ -68,7 +68,7 @@ thirdPartyAudit.excludes = [
]
// pulled in as external dependency to work on java 9
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.excludes += [
'com.sun.activation.registries.MailcapParseException',
'javax.activation.ActivationDataFlavor',