lucene/gradle/validation/jar-checks.gradle

394 lines
14 KiB
Groovy

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This adds validation of project dependencies:
// 1) license file
// 2) notice file
// 3) checksum validation/ generation.
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.codec.digest.MessageDigestAlgorithms
// This should be false only for debugging.
def failOnError = true
// We're using commons-codec for computing checksums.
buildscript {
repositories {
mavenCentral()
}
dependencies {
classpath 'commons-codec:commons-codec:1.13'
}
}
// Configure license checksum folder for top-level projects.
// (The file("licenses") inside the configure scope resolves
// relative to the current project so they're not the same).
configure(project(":lucene")) {
ext.licensesDir = file("licenses")
}
configure(project(":solr")) {
ext.licensesDir = file("licenses")
}
// All known license types. If 'noticeOptional' is true then
// the notice file must accompany the license.
def licenseTypes = [
"ASL" : [name: "Apache Software License 2.0"],
"BSD" : [name: "Berkeley Software Distribution"],
//BSD like just means someone has taken the BSD license and put in their name, copyright, or it's a very similar license.
"BSD_LIKE": [name: "BSD like license"],
"CDDL" : [name: "Common Development and Distribution License", noticeOptional: true],
"CPL" : [name: "Common Public License"],
"EPL" : [name: "Eclipse Public License Version 1.0", noticeOptional: true],
"MIT" : [name: "Massachusetts Institute of Tech. License", noticeOptional: true],
"MPL" : [name: "Mozilla Public License", noticeOptional: true /* NOT SURE on the required notice */],
"PD" : [name: "Public Domain", noticeOptional: true],
"SUN" : [name: "Sun Open Source License", noticeOptional: true],
"COMPOUND": [name: "Compound license (details in NOTICE file)."],
]
allprojects {
task licenses() {
group = 'Dependency validation'
description = "Apply all dependency/ license checks."
}
check.dependsOn(licenses)
}
subprojects {
// Configure jarValidation configuration for all projects. Any dependency
// declared on this configuration (or any configuration it extends from) will
// be verified.
configurations {
jarValidation
}
// For Java projects, add all dependencies from the following configurations
// to jar validation
plugins.withType(JavaPlugin) {
configurations {
jarValidation {
extendsFrom runtimeClasspath
extendsFrom compileClasspath
extendsFrom testRuntimeClasspath
extendsFrom testCompileClasspath
}
}
}
// Collects dependency JAR information for a project and saves it in
// project.ext.jarInfos. Each dependency has a map of attributes
// which make it easier to process it later on (name, hash, origin module,
// see the code below for details).
task collectJarInfos() {
dependsOn configurations.jarValidation
doFirst {
def isSolr = project.path.startsWith(":solr")
// When gradle resolves a configuration it applies exclude rules from inherited configurations
// globally (this seems like a bug to me). So we process each inherited configuration independently
// but make sure there are no other dependencies on jarValidation itself.
if (!configurations.jarValidation.dependencies.isEmpty()) {
throw new GradleException("jarValidation must only inherit from other configurations (can't have its own dependencies).")
}
def excludeRules = configurations.jarValidation.excludeRules
ArrayDeque<ResolvedDependency> queue = new ArrayDeque<>()
configurations.jarValidation.extendsFrom.each { conf ->
if (excludeRules) {
conf = configurations.detachedConfiguration().extendsFrom(conf)
conf.excludeRules = excludeRules
}
if (conf.canBeResolved) {
queue.addAll(conf.resolvedConfiguration.firstLevelModuleDependencies)
}
}
def visited = new HashSet<>()
def infos = []
while (!queue.isEmpty()) {
def dep = queue.removeFirst()
// Skip any artifacts from other Solr modules (they will be resolved there).
if (dep.moduleGroup == "org.apache.solr") {
continue
}
// Skip any artifacts from Lucene modules.
if (dep.moduleGroup.startsWith("org.apache.lucene")) {
// ... but process their transitive dependencies for Solr compatibility.
if (isSolr) {
queue.addAll(dep.children)
}
} else {
queue.addAll(dep.children)
dep.moduleArtifacts.each { resolvedArtifact ->
def file = resolvedArtifact.file
if (visited.add(file)) {
infos.add([
name : resolvedArtifact.name,
jarName : file.toPath().getFileName().toString(),
path : file,
module : resolvedArtifact.moduleVersion,
checksum : provider { new DigestUtils(MessageDigestAlgorithms.SHA_1).digestAsHex(file).trim() },
// We keep track of the files referenced by this dependency (sha, license, notice, etc.)
// so that we can determine unused dangling files later on.
referencedFiles: []
])
}
}
}
}
project.ext.jarInfos = infos.sort {a, b -> "${a.module}".compareTo("${b.module}")}
// jarInfos.each { info -> println "${info.module}" }
}
}
// Verifies that each JAR has a corresponding checksum and that it matches actual JAR available for this dependency.
task validateJarChecksums() {
group = 'Dependency validation'
description = "Validate checksums of dependencies"
dependsOn collectJarInfos
doLast {
def errors = []
jarInfos.each { dep ->
def expectedChecksumFile = file("${licensesDir}/${dep.jarName}.sha1")
if (!expectedChecksumFile.exists()) {
errors << "Dependency checksum missing ('${dep.module}'), expected it at: ${expectedChecksumFile}"
} else {
dep.referencedFiles += expectedChecksumFile
def expected = expectedChecksumFile.getText("UTF-8").trim()
def actual = dep.checksum.get()
if (expected.compareToIgnoreCase(actual) != 0) {
errors << "Dependency checksum mismatch ('${dep.module}'), expected it to be: ${expected}, but was: ${actual}"
} else {
logger.log(LogLevel.INFO, "Dependency checksum OK ('${dep.module}')")
}
}
}
if (errors) {
def msg = "Dependency checksum validation failed:\n - " + errors.join("\n - ")
if (failOnError) {
throw new GradleException(msg)
} else {
logger.log(LogLevel.WARN, "WARNING: ${msg}")
}
}
}
}
// Locate the set of license file candidates for this dependency. We
// search for [jar-or-prefix]-LICENSE-[type].txt
// where 'jar-or-prefix' can be any '-'-delimited prefix of the dependency JAR's name.
// So for 'commons-io' it can be 'commons-io-LICENSE-foo.txt' or
// 'commons-LICENSE.txt'
task validateJarLicenses() {
group = 'Dependency validation'
description = "Validate license and notice files of dependencies"
dependsOn collectJarInfos
doLast {
def errors = []
jarInfos.each { dep ->
def baseName = dep.name
def found = []
def candidates = []
while (true) {
candidates += file("${licensesDir}/${baseName}-LICENSE-[type].txt")
found += fileTree(dir: licensesDir, include: "${baseName}-LICENSE-*.txt").files
def prefix = baseName.replaceAll(/[\-][^-]+$/, "")
if (found || prefix == baseName) {
break
}
baseName = prefix
}
if (found.size() == 0) {
errors << "License file missing ('${dep.module}'), expected it at: ${candidates.join(" or ")}," +
" where [type] can be any of ${licenseTypes.keySet()}."
} else if (found.size() > 1) {
errors << "Multiple license files matching for ('${dep.module}'): ${found.join(", ")}"
} else {
def licenseFile = found.get(0)
dep.referencedFiles += licenseFile
def m = (licenseFile.name =~ /LICENSE-(.+)\.txt$/)
if (!m) throw new GradleException("License file name doesn't contain license type?: ${licenseFile.name}")
def licenseName = m[0][1]
def licenseType = licenseTypes[licenseName]
if (!licenseType) {
errors << "Unknown license type suffix for ('${dep.module}'): ${licenseFile} (must be one of ${licenseTypes.keySet()})"
} else {
logger.log(LogLevel.INFO, "Dependency license file OK ('${dep.module}'): " + licenseName)
// Look for sibling NOTICE file.
def noticeFile = file(licenseFile.path.replaceAll(/\-LICENSE-.+/, "-NOTICE.txt"))
if (noticeFile.exists()) {
dep.referencedFiles += noticeFile
logger.log(LogLevel.INFO, "Dependency notice file OK ('${dep.module}'): " + noticeFile)
} else if (!licenseType.noticeOptional) {
errors << "Notice file missing for ('${dep.module}'), expected it at: ${noticeFile}"
}
}
}
}
if (errors) {
def msg = "Certain license/ notice files are missing:\n - " + errors.join("\n - ")
if (failOnError) {
throw new GradleException(msg)
} else {
logger.log(LogLevel.WARN, "WARNING: ${msg}")
}
}
}
}
licenses.dependsOn validateJarChecksums, validateJarLicenses
}
// Add top-project level tasks validating dangling files
// and regenerating dependency checksums.
configure([project(":solr"), project(":lucene"),]) {
def validationTasks = subprojects.collectMany { it.tasks.matching { it.name == "licenses" } }
def jarInfoTasks = subprojects.collectMany { it.tasks.matching { it.name == "collectJarInfos" } }
// Update dependency checksums.
task updateLicenses() {
group = 'Dependency validation'
description = "Write or update checksums of dependencies. May need to run gradlew --write-locks first if changing dependency versions"
dependsOn jarInfoTasks
doLast {
licensesDir.mkdirs()
// Clean any previous checksums. In theory we wouldn't have to do it --
// dangling files from any previous JARs would be reported;
// it automates the process of updating versions and makes it easier though so
// why not.
project.delete fileTree(licensesDir, {
include "*.sha1"
exclude checkDanglingLicenseFiles.ext.exclude
})
def updated = []
jarInfoTasks.collectMany { task -> task.project.jarInfos }.each { dep ->
def expectedChecksumFile = file("${licensesDir}/${dep.jarName}.sha1")
def actual = dep.checksum.get()
if (expectedChecksumFile.exists()) {
def expected = expectedChecksumFile.getText("UTF-8").trim()
if (expected.compareToIgnoreCase(actual) == 0) {
return;
}
}
updated += "Updated checksum ('${dep.module}'): ${expectedChecksumFile}"
expectedChecksumFile.write(actual + "\n", "UTF-8")
}
updated.sort().each { line -> logger.log(LogLevel.LIFECYCLE, line) }
}
}
// Any validation task must run after all updates have been applied.
// We add an ordering constraint that any validation task (or its dependency subgraph)
// must run after updateLicenses
validationTasks
.collectMany { task -> [task, task.dependsOn]}
.flatten()
.each { task ->
task.mustRunAfter updateLicenses
}
// Check for dangling files in the licenses folder.
task checkDanglingLicenseFiles() {
dependsOn validationTasks
ext {
exclude = []
}
doFirst {
def allReferenced = validationTasks.collectMany { task ->
task.project.jarInfos.collectMany { it.referencedFiles }
}.collect { it.toString() }
def patterns = ext.exclude
def allExisting = fileTree(licensesDir, {
exclude patterns
}).files.collect { it.toString() }
def dangling = (allExisting - allReferenced).sort()
if (dangling) {
gradle.buildFinished {
logger.warn("WARNING: there were unreferenced files under license folder:\n - ${dangling.join("\n - ")}")
}
}
}
}
licenses.dependsOn checkDanglingLicenseFiles
}
// Exclude files that are not a result of direct dependencies but have to be there.
// It would be probably better to move non-dependency licenses into the actual project
// where they're used and only assemble them for the distribution package.
configure(project(":lucene")) {
checkDanglingLicenseFiles {
exclude += [
"elegant-icon-font-*",
"ant-*",
"ivy-*",
]
}
}
configure(project(":solr")) {
checkDanglingLicenseFiles {
exclude += [
"README.committers.txt",
// solr-ref-guide compilation-only dependencies.
"android-json-*",
"ant-*",
"asciidoctor-ant-*",
"jsoup-*",
"junit4-ant-*",
"slf4j-simple-*",
"start.jar.sha1"
]
}
}
// solr-ref-guide doesn't contribute any JARs to dependency checks.
configure(project(":solr:solr-ref-guide")) {
configurations {
jarValidation {
exclude group: "*"
}
}
}