honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix UP-TO-DATE check for some tasks 

* third party audit
* jar hell
* properties file write in buildSrc
* license headers
This commit is contained in:
Nik Everett 2016-03-29 17:51:48 -04:00
parent 0eb1a816c8
commit 5968c4d9d1
4 changed files with 117 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
buildSrc/build.gradle
View File

@ -80,6 +80,7 @@ dependencies {
File tempPropertiesFile = new File(project.buildDir, "version.properties")
task writeVersionProperties {
inputs.properties(props)
outputs.file(tempPropertiesFile)
doLast {
OutputStream stream = Files.newOutputStream(tempPropertiesFile.toPath());
try {

9
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy
View File

@ -21,7 +21,6 @@ package org.elasticsearch.gradle.precommit
import org.elasticsearch.gradle.LoggedExec
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.OutputFile
/**
@ -35,14 +34,12 @@ public class JarHellTask extends LoggedExec {
* inputs (ie the jars/class files).
*/
@OutputFile
public File successMarker = new File(project.buildDir, 'markers/jarHell')
/** The classpath to run jarhell check on, defaults to the test runtime classpath */
@InputFile
public FileCollection classpath = project.sourceSets.test.runtimeClasspath
File successMarker = new File(project.buildDir, 'markers/jarHell')
public JarHellTask() {
project.afterEvaluate {
FileCollection classpath = project.sourceSets.test.runtimeClasspath
inputs.files(classpath)
dependsOn(classpath)
description = "Runs CheckJarHell on ${classpath}"
executable = new File(project.javaHome, 'bin/java')

43
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy
View File

@ -22,6 +22,8 @@ import org.apache.rat.anttasks.Report
import org.apache.rat.anttasks.SubstringLicenseMatcher
import org.apache.rat.license.SimpleLicenseFamily
import org.elasticsearch.gradle.AntTask
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.SourceSet
import java.nio.file.Files
@ -33,8 +35,27 @@ import java.nio.file.Files
*/
public class LicenseHeadersTask extends AntTask {
@OutputFile
File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log')
private List<FileCollection> javaFiles
LicenseHeadersTask() {
description = "Checks sources for missing, incorrect, or unacceptable license headers"
// Delay resolving the dependencies until after evaluation so we pick up generated sources
project.afterEvaluate {
List<FileCollection> javaFiles = project.sourceSets.collect({it.allJava})
setJavaFiles(javaFiles)
inputs.files(javaFiles)
}
}
/**
* Set the source sets this task processes. Should only be used by the afterEvaluate closure
* in the constructor.
*/
protected void setJavaFiles(List<FileCollection> javaFiles) {
this.javaFiles = javaFiles
}
@Override
@ -43,17 +64,13 @@ public class LicenseHeadersTask extends AntTask {
ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher)
ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily)
// create a file for the log to go to under reports/
File reportDir = new File(project.buildDir, "reports/licenseHeaders")
reportDir.mkdirs()
File reportFile = new File(reportDir, "rat.log")
Files.deleteIfExists(reportFile.toPath())
// run rat, going to the file
List<FileCollection> input = javaFiles
ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
// checks all the java sources (allJava)
for (SourceSet set : project.sourceSets) {
for (File dir : set.allJava.srcDirs) {
for (FileCollection dirSet : input) {
for (File dir: dirSet.srcDirs) {
// sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
if (dir.exists()) {
ant.fileset(dir: dir)
@ -85,12 +102,12 @@ public class LicenseHeadersTask extends AntTask {
// parsers generated by antlr
pattern(substring: "ANTLR GENERATED CODE")
}
// approved categories
approvedLicense(familyName: "Apache")
approvedLicense(familyName: "Generated")
approvedLicense(familyName: "Generated")
}
// check the license file for any errors, this should be fast.
boolean zeroUnknownLicenses = false
boolean foundProblemsWithFiles = false
@ -98,12 +115,12 @@ public class LicenseHeadersTask extends AntTask {
if (line.startsWith("0 Unknown Licenses")) {
zeroUnknownLicenses = true
}
if (line.startsWith(" !")) {
foundProblemsWithFiles = true
}
}
if (zeroUnknownLicenses == false || foundProblemsWithFiles) {
// print the unapproved license section, usually its all you need to fix problems.
int sectionNumber = 0

126
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
View File

@ -27,6 +27,9 @@ import org.apache.tools.ant.Project;
import org.elasticsearch.gradle.AntTask;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
@ -40,17 +43,78 @@ import java.util.regex.Pattern;
* Basic static checking to keep tabs on third party JARs
*/
public class ThirdPartyAuditTask extends AntTask {
// patterns for classes to exclude, because we understand their issues
private String[] excludes = new String[0];
private List<String> excludes = [];
/**
* Input for the task. Set javadoc for {#link getJars} for more.
*/
private FileCollection jars;
/**
* Classpath against which to run the third patty audit.
*/
private FileCollection classpath;
/**
* We use a simple "marker" file that we touch when the task succeeds
* as the task output. This is compared against the modified time of the
* inputs (ie the jars/class files).
*/
@OutputFile
File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit')
ThirdPartyAuditTask() {
// we depend on this because its the only reliable configuration
// this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price.
dependsOn(project.configurations.testCompile);
description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'";
project.afterEvaluate {
Configuration configuration = project.configurations.findByName('runtime');
if (configuration == null) {
// some projects apparently do not have 'runtime'? what a nice inconsistency,
// basically only serves to waste time in build logic!
configuration = project.configurations.findByName('testCompile');
}
assert configuration != null;
setClasspath(configuration)
// we only want third party dependencies.
FileCollection jars = configuration.fileCollection({ dependency ->
dependency.group.startsWith("org.elasticsearch") == false
});
// we don't want provided dependencies, which we have already scanned. e.g. don't
// scan ES core's dependencies for every single plugin
Configuration provided = project.configurations.findByName('provided')
if (provided != null) {
jars -= provided
}
setJars(jars)
inputs.files(jars)
onlyIf { jars.isEmpty() == false }
}
}
/**
* Set the jars to process. This should only be called by the afterEvaluate
* closure in the constructor.
*/
protected void setJars(FileCollection jars) {
this.jars = jars
}
/**
* Set the classpath against which to run the third party audit. This
* should only be called by the afterEvaluate closure in the constructor.
*/
protected void setClasspath(FileCollection classpath) {
this.classpath = classpath
}
/**
* classes that should be excluded from the scan,
* e.g. because we know what sheisty stuff those particular classes are up to.
@ -61,21 +125,22 @@ public class ThirdPartyAuditTask extends AntTask {
throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!");
}
}
excludes = classes;
excludes = classes.sort();
}
/**
* Returns current list of exclusions.
*/
public String[] getExcludes() {
@Input
public List<String> getExcludes() {
return excludes;
}
// yes, we parse Uwe Schindler's errors to find missing classes, and to keep a continuous audit. Just don't let him know!
static final Pattern MISSING_CLASS_PATTERN =
Pattern.compile(/WARNING: The referenced class '(.*)' cannot be loaded\. Please fix the classpath\!/);
static final Pattern VIOLATION_PATTERN =
static final Pattern VIOLATION_PATTERN =
Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/);
// we log everything and capture errors and handle them with our whitelist
@ -124,32 +189,8 @@ public class ThirdPartyAuditTask extends AntTask {
@Override
protected void runAnt(AntBuilder ant) {
Configuration configuration = project.configurations.findByName('runtime');
if (configuration == null) {
// some projects apparently do not have 'runtime'? what a nice inconsistency,
// basically only serves to waste time in build logic!
configuration = project.configurations.findByName('testCompile');
}
assert configuration != null;
ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask);
// we only want third party dependencies.
FileCollection jars = configuration.fileCollection({ dependency ->
dependency.group.startsWith("org.elasticsearch") == false
});
// we don't want provided dependencies, which we have already scanned. e.g. don't
// scan ES core's dependencies for every single plugin
Configuration provided = project.configurations.findByName('provided');
if (provided != null) {
jars -= provided;
}
// no dependencies matched, we are done
if (jars.isEmpty()) {
return;
}
// print which jars we are going to scan, always
// this is not the time to try to be succinct! Forbidden will print plenty on its own!
Set<String> names = new TreeSet<>();
@ -171,26 +212,23 @@ public class ThirdPartyAuditTask extends AntTask {
}
// convert exclusion class names to binary file names
String[] excludedFiles = new String[excludes.length];
for (int i = 0; i < excludes.length; i++) {
excludedFiles[i] = excludes[i].replace('.', '/') + ".class";
}
Set<String> excludedSet = new TreeSet<>(Arrays.asList(excludedFiles));
List<String> excludedFiles = excludes.collect {it.replace('.', '/') + ".class"}
Set<String> excludedSet = new TreeSet<>(excludedFiles);
// jarHellReprise
Set<String> sheistySet = getSheistyClasses(tmpDir.toPath());
try {
try {
ant.thirdPartyAudit(internalRuntimeForbidden: false,
failOnUnsupportedJava: false,
failOnMissingClasses: false,
signaturesFile: new File(getClass().getResource('/forbidden/third-party-audit.txt').toURI()),
classpath: configuration.asPath) {
classpath: classpath.asPath) {
fileset(dir: tmpDir)
}
} catch (BuildException ignore) {}
EvilLogger evilLogger = null;
EvilLogger evilLogger = null;
for (BuildListener listener : ant.project.getBuildListeners()) {
if (listener instanceof EvilLogger) {
evilLogger = (EvilLogger) listener;
@ -228,6 +266,8 @@ public class ThirdPartyAuditTask extends AntTask {
// clean up our mess (if we succeed)
ant.delete(dir: tmpDir.getAbsolutePath());
successMarker.setText("", 'UTF-8')
}
/**
@ -235,11 +275,11 @@ public class ThirdPartyAuditTask extends AntTask {
*/
private Set<String> getSheistyClasses(Path root) {
// system.parent = extensions loader.
// note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!).
// note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!).
// but groovy/gradle needs to work at all first!
ClassLoader ext = ClassLoader.getSystemClassLoader().getParent();
assert ext != null;
Set<String> sheistySet = new TreeSet<>();
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override