From 5968c4d9d10241ae7effc13172af1abeaf18c331 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 29 Mar 2016 17:51:48 -0400 Subject: [PATCH] Fix UP-TO-DATE check for some tasks * third party audit * jar hell * properties file write in buildSrc * license headers --- buildSrc/build.gradle | 1 + .../gradle/precommit/JarHellTask.groovy | 9 +- .../precommit/LicenseHeadersTask.groovy | 43 ++++-- .../precommit/ThirdPartyAuditTask.groovy | 126 ++++++++++++------ 4 files changed, 117 insertions(+), 62 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index f8d80679504..e36451311e7 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -80,6 +80,7 @@ dependencies { File tempPropertiesFile = new File(project.buildDir, "version.properties") task writeVersionProperties { inputs.properties(props) + outputs.file(tempPropertiesFile) doLast { OutputStream stream = Files.newOutputStream(tempPropertiesFile.toPath()); try { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy index 2873fbd4df5..f8eb0a63c96 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy @@ -21,7 +21,6 @@ package org.elasticsearch.gradle.precommit import org.elasticsearch.gradle.LoggedExec import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.OutputFile /** @@ -35,14 +34,12 @@ public class JarHellTask extends LoggedExec { * inputs (ie the jars/class files). */ @OutputFile - public File successMarker = new File(project.buildDir, 'markers/jarHell') - - /** The classpath to run jarhell check on, defaults to the test runtime classpath */ - @InputFile - public FileCollection classpath = project.sourceSets.test.runtimeClasspath + File successMarker = new File(project.buildDir, 'markers/jarHell') public JarHellTask() { project.afterEvaluate { + FileCollection classpath = project.sourceSets.test.runtimeClasspath + inputs.files(classpath) dependsOn(classpath) description = "Runs CheckJarHell on ${classpath}" executable = new File(project.javaHome, 'bin/java') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy index 39cf55c905b..0af02188efb 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy @@ -22,6 +22,8 @@ import org.apache.rat.anttasks.Report import org.apache.rat.anttasks.SubstringLicenseMatcher import org.apache.rat.license.SimpleLicenseFamily import org.elasticsearch.gradle.AntTask +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.SourceSet import java.nio.file.Files @@ -33,8 +35,27 @@ import java.nio.file.Files */ public class LicenseHeadersTask extends AntTask { + @OutputFile + File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log') + + private List javaFiles + LicenseHeadersTask() { description = "Checks sources for missing, incorrect, or unacceptable license headers" + // Delay resolving the dependencies until after evaluation so we pick up generated sources + project.afterEvaluate { + List javaFiles = project.sourceSets.collect({it.allJava}) + setJavaFiles(javaFiles) + inputs.files(javaFiles) + } + } + + /** + * Set the source sets this task processes. Should only be used by the afterEvaluate closure + * in the constructor. + */ + protected void setJavaFiles(List javaFiles) { + this.javaFiles = javaFiles } @Override @@ -43,17 +64,13 @@ public class LicenseHeadersTask extends AntTask { ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher) ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily) - // create a file for the log to go to under reports/ - File reportDir = new File(project.buildDir, "reports/licenseHeaders") - reportDir.mkdirs() - File reportFile = new File(reportDir, "rat.log") Files.deleteIfExists(reportFile.toPath()) - + // run rat, going to the file + List input = javaFiles ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) { - // checks all the java sources (allJava) - for (SourceSet set : project.sourceSets) { - for (File dir : set.allJava.srcDirs) { + for (FileCollection dirSet : input) { + for (File dir: dirSet.srcDirs) { // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main... if (dir.exists()) { ant.fileset(dir: dir) @@ -85,12 +102,12 @@ public class LicenseHeadersTask extends AntTask { // parsers generated by antlr pattern(substring: "ANTLR GENERATED CODE") } - + // approved categories approvedLicense(familyName: "Apache") - approvedLicense(familyName: "Generated") + approvedLicense(familyName: "Generated") } - + // check the license file for any errors, this should be fast. boolean zeroUnknownLicenses = false boolean foundProblemsWithFiles = false @@ -98,12 +115,12 @@ public class LicenseHeadersTask extends AntTask { if (line.startsWith("0 Unknown Licenses")) { zeroUnknownLicenses = true } - + if (line.startsWith(" !")) { foundProblemsWithFiles = true } } - + if (zeroUnknownLicenses == false || foundProblemsWithFiles) { // print the unapproved license section, usually its all you need to fix problems. int sectionNumber = 0 diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy index 5d06103789f..84cae39d8f6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy @@ -27,6 +27,9 @@ import org.apache.tools.ant.Project; import org.elasticsearch.gradle.AntTask; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.OutputFile import java.nio.file.FileVisitResult; import java.nio.file.Files; @@ -40,17 +43,78 @@ import java.util.regex.Pattern; * Basic static checking to keep tabs on third party JARs */ public class ThirdPartyAuditTask extends AntTask { - + // patterns for classes to exclude, because we understand their issues - private String[] excludes = new String[0]; - + private List excludes = []; + + /** + * Input for the task. Set javadoc for {#link getJars} for more. + */ + private FileCollection jars; + + /** + * Classpath against which to run the third patty audit. + */ + private FileCollection classpath; + + /** + * We use a simple "marker" file that we touch when the task succeeds + * as the task output. This is compared against the modified time of the + * inputs (ie the jars/class files). + */ + @OutputFile + File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit') + ThirdPartyAuditTask() { // we depend on this because its the only reliable configuration // this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price. dependsOn(project.configurations.testCompile); description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"; + + + project.afterEvaluate { + Configuration configuration = project.configurations.findByName('runtime'); + if (configuration == null) { + // some projects apparently do not have 'runtime'? what a nice inconsistency, + // basically only serves to waste time in build logic! + configuration = project.configurations.findByName('testCompile'); + } + assert configuration != null; + setClasspath(configuration) + + // we only want third party dependencies. + FileCollection jars = configuration.fileCollection({ dependency -> + dependency.group.startsWith("org.elasticsearch") == false + }); + + // we don't want provided dependencies, which we have already scanned. e.g. don't + // scan ES core's dependencies for every single plugin + Configuration provided = project.configurations.findByName('provided') + if (provided != null) { + jars -= provided + } + setJars(jars) + inputs.files(jars) + onlyIf { jars.isEmpty() == false } + } } - + + /** + * Set the jars to process. This should only be called by the afterEvaluate + * closure in the constructor. + */ + protected void setJars(FileCollection jars) { + this.jars = jars + } + + /** + * Set the classpath against which to run the third party audit. This + * should only be called by the afterEvaluate closure in the constructor. + */ + protected void setClasspath(FileCollection classpath) { + this.classpath = classpath + } + /** * classes that should be excluded from the scan, * e.g. because we know what sheisty stuff those particular classes are up to. @@ -61,21 +125,22 @@ public class ThirdPartyAuditTask extends AntTask { throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!"); } } - excludes = classes; + excludes = classes.sort(); } - + /** * Returns current list of exclusions. */ - public String[] getExcludes() { + @Input + public List getExcludes() { return excludes; } // yes, we parse Uwe Schindler's errors to find missing classes, and to keep a continuous audit. Just don't let him know! static final Pattern MISSING_CLASS_PATTERN = Pattern.compile(/WARNING: The referenced class '(.*)' cannot be loaded\. Please fix the classpath\!/); - - static final Pattern VIOLATION_PATTERN = + + static final Pattern VIOLATION_PATTERN = Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/); // we log everything and capture errors and handle them with our whitelist @@ -124,32 +189,8 @@ public class ThirdPartyAuditTask extends AntTask { @Override protected void runAnt(AntBuilder ant) { - Configuration configuration = project.configurations.findByName('runtime'); - if (configuration == null) { - // some projects apparently do not have 'runtime'? what a nice inconsistency, - // basically only serves to waste time in build logic! - configuration = project.configurations.findByName('testCompile'); - } - assert configuration != null; ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask); - // we only want third party dependencies. - FileCollection jars = configuration.fileCollection({ dependency -> - dependency.group.startsWith("org.elasticsearch") == false - }); - - // we don't want provided dependencies, which we have already scanned. e.g. don't - // scan ES core's dependencies for every single plugin - Configuration provided = project.configurations.findByName('provided'); - if (provided != null) { - jars -= provided; - } - - // no dependencies matched, we are done - if (jars.isEmpty()) { - return; - } - // print which jars we are going to scan, always // this is not the time to try to be succinct! Forbidden will print plenty on its own! Set names = new TreeSet<>(); @@ -171,26 +212,23 @@ public class ThirdPartyAuditTask extends AntTask { } // convert exclusion class names to binary file names - String[] excludedFiles = new String[excludes.length]; - for (int i = 0; i < excludes.length; i++) { - excludedFiles[i] = excludes[i].replace('.', '/') + ".class"; - } - Set excludedSet = new TreeSet<>(Arrays.asList(excludedFiles)); + List excludedFiles = excludes.collect {it.replace('.', '/') + ".class"} + Set excludedSet = new TreeSet<>(excludedFiles); // jarHellReprise Set sheistySet = getSheistyClasses(tmpDir.toPath()); - try { + try { ant.thirdPartyAudit(internalRuntimeForbidden: false, failOnUnsupportedJava: false, failOnMissingClasses: false, signaturesFile: new File(getClass().getResource('/forbidden/third-party-audit.txt').toURI()), - classpath: configuration.asPath) { + classpath: classpath.asPath) { fileset(dir: tmpDir) } } catch (BuildException ignore) {} - EvilLogger evilLogger = null; + EvilLogger evilLogger = null; for (BuildListener listener : ant.project.getBuildListeners()) { if (listener instanceof EvilLogger) { evilLogger = (EvilLogger) listener; @@ -228,6 +266,8 @@ public class ThirdPartyAuditTask extends AntTask { // clean up our mess (if we succeed) ant.delete(dir: tmpDir.getAbsolutePath()); + + successMarker.setText("", 'UTF-8') } /** @@ -235,11 +275,11 @@ public class ThirdPartyAuditTask extends AntTask { */ private Set getSheistyClasses(Path root) { // system.parent = extensions loader. - // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!). + // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!). // but groovy/gradle needs to work at all first! ClassLoader ext = ClassLoader.getSystemClassLoader().getParent(); assert ext != null; - + Set sheistySet = new TreeSet<>(); Files.walkFileTree(root, new SimpleFileVisitor() { @Override