diff --git a/build.gradle b/build.gradle index 0f4768de19a..75756fdc56b 100644 --- a/build.gradle +++ b/build.gradle @@ -51,6 +51,7 @@ apply from: file('gradle/validation/git-status.gradle') apply from: file('gradle/validation/versions-props-sorted.gradle') apply from: file('gradle/validation/validate-source-patterns.gradle') apply from: file('gradle/validation/config-file-sanity.gradle') +apply from: file('gradle/validation/rat-sources.gradle') // Additional development aids. apply from: file('gradle/maven/maven-local.gradle') diff --git a/gradle/validation/precommit.gradle b/gradle/validation/precommit.gradle index 88f9b996e74..f68b9c6597f 100644 --- a/gradle/validation/precommit.gradle +++ b/gradle/validation/precommit.gradle @@ -22,6 +22,7 @@ configure(rootProject) { "forbiddenApisTest", "licenses", "javadoc", + "rat", ]} } } diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle new file mode 100644 index 00000000000..90291e67ef3 --- /dev/null +++ b/gradle/validation/rat-sources.gradle @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import groovy.xml.NamespaceBuilder + +configure(rootProject) { + configurations { + rat + } + + dependencies { + rat "org.apache.rat:apache-rat" + } +} + +allprojects { + task("rat", type: RatTask) { + group = 'Verification' + description = 'Runs Apache Rat checks.' + } + + if (project == rootProject) { + rat { + includes += [ + "buildSrc/**/*.java", + "lucene/tools/forbiddenApis/**", + "lucene/tools/prettify/**", + // "dev-tools/**" + ] + excludes += [ + // Unclear if this needs ASF header, depends on how much was copied from ElasticSearch + "**/ErrorReportingTestListener.java" + ] + } + } + + if (project.path == ":lucene:analysis:common") { + rat { + srcExcludes += [ + "**/*.aff", + "**/*.dic", + "**/charfilter/*.htm*", + "**/*LuceneResourcesWikiPage.html" + ] + } + } + + if (project.path == ":lucene:analysis:kuromoji") { + rat { + srcExcludes += [ + // whether rat detects this as binary or not is platform dependent?! + "**/bocchan.utf-8" + ] + } + } + + if (project.path == ":lucene:analysis:opennlp") { + rat { + excludes += [ + "src/tools/test-model-data/*.txt", + ] + } + } + + if (project.path == ":lucene:highlighter") { + rat { + srcExcludes += [ + "**/CambridgeMA.utf8" + ] + } + } + + if (project.path == ":lucene:suggest") { + rat { + srcExcludes += [ + "**/Top50KWiki.utf8", + "**/stop-snowball.txt" + ] + } + } + + if (project.path == ":lucene:tools") { + rat { + includes += [ + "forbiddenApis/**", + "prettify/**", + // If/when :lucene:tools becomes a gradle project, then the following line will fail + // It is safe to remove it, but also remove the check for lucene/tools in rootProject above + "javadoc/ecj.javadocs.prefs" + ] + } + } + + + if (project.path == ":solr:core") { + rat { + srcExcludes += [ + "**/htmlStripReaderTest.html" + ] + } + } + + if (project.path == ":solr:webapp") { + rat { + includes = [ "**" ] + excludes += [ + "web/img/**", + "*.iml", + "build.gradle", + "build/**", + ] + } + } +} + +// Structure inspired by existing task from Apache Kafka, heavily modified since then. +class RatTask extends DefaultTask { + @Input + List includes = [ + "*.xml", + "src/tools/**" + ] + + @Input + List excludes = [] + + @Input + List srcExcludes = [ + "**/TODO", + "**/*.txt", + "**/*.iml", + "**/*.gradle", + "build/**" + ] + + @OutputFile + def xmlReport = new File(new File(project.buildDir, 'rat'), 'rat-report.xml') + + def generateXmlReport() { + def uri = 'antlib:org.apache.rat.anttasks' + def ratClasspath = project.rootProject.configurations.rat.asPath + ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', uri: uri, classpath: ratClasspath) + + def rat = NamespaceBuilder.newInstance(ant, uri) + rat.report(format: 'xml', reportFile: xmlReport, addDefaultLicenseMatchers: true) { + ant.fileset(dir: "${project.projectDir}") { + includes.each { pattern -> ant.include(name: pattern) } + excludes.each { pattern -> ant.exclude(name: pattern) } + } + + if (project.plugins.findPlugin(JavaPlugin)) { + [ + project.sourceSets.main.java.srcDirs, + project.sourceSets.test.java.srcDirs, + ].flatten().each { srcLocation -> + ant.fileset(dir: srcLocation, erroronmissingdir: false) { + srcExcludes.each { pattern -> ant.exclude(name: pattern) } + } + } + + [ + project.sourceSets.main.resources.srcDirs + ].flatten().each { srcLocation -> + ant.fileset(dir: srcLocation, erroronmissingdir: false) { + ant.include(name: "META-INF/**") + } + } + } + + // The license rules below were manually copied from lucene/common-build.xml, there is currently no mechanism to sync them + + // BSD 4-clause stuff (is disallowed below) + substringMatcher(licenseFamilyCategory: "BSD4 ", licenseFamilyName: "Original BSD License (with advertising clause)") { + pattern(substring: "All advertising materials") + } + + // BSD-like stuff + substringMatcher(licenseFamilyCategory: "BSD ", licenseFamilyName: "Modified BSD License") { + // brics automaton + pattern(substring: "Copyright (c) 2001-2009 Anders Moeller") + // snowball + pattern(substring: "Copyright (c) 2001, Dr Martin Porter") + // UMASS kstem + pattern(substring: "THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS") + // Egothor + pattern(substring: "Egothor Software License version 1.00") + // JaSpell + pattern(substring: "Copyright (c) 2005 Bruno Martins") + // d3.js + pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS") + // highlight.js + pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS") + } + + // MIT-like + substringMatcher(licenseFamilyCategory: "MIT ", licenseFamilyName:"Modified BSD License") { + // ICU license + pattern(substring: "Permission is hereby granted, free of charge, to any person obtaining a copy") + } + + // Apache + substringMatcher(licenseFamilyCategory: "AL ", licenseFamilyName: "Apache") { + pattern(substring: "Licensed to the Apache Software Foundation (ASF) under") + // this is the old - school one under some files + pattern(substring: 'Licensed under the Apache License, Version 2.0 (the "License")') + } + + substringMatcher(licenseFamilyCategory: "GEN ", licenseFamilyName: "Generated") { + // svg files generated by gnuplot + pattern(substring: "Produced by GNUPLOT") + // snowball stemmers generated by snowball compiler + pattern(substring: "This file was generated automatically by the Snowball to Java compiler") + // parsers generated by antlr + pattern(substring: "ANTLR GENERATED CODE") + } + + approvedLicense(familyName: "Apache") + approvedLicense(familyName: "The MIT License") + approvedLicense(familyName: "Modified BSD License") + approvedLicense(familyName: "Generated") + } + } + + def printUnknownFiles() { + def ratXml = new XmlParser().parse(xmlReport) + def errors = [] + ratXml.resource.each { resource -> + if (resource.'license-approval'.@name[0] == "false") { + errors << "Unknown license: ${resource.@name}" + } + } + if (errors) { + throw new GradleException("Found " + errors.size() + " file(s) with errors:\n" + + errors.collect{ msg -> " - ${msg}" }.join("\n")) + } + } + + @TaskAction + def rat() { + def origEncoding = System.getProperty("file.encoding") + try { + generateXmlReport() + printUnknownFiles() + } finally { + if (System.getProperty("file.encoding") != origEncoding) { + throw new GradleException("Insane: rat changed file.encoding to ${System.getProperty('file.encoding')}?") + } + } + } +} diff --git a/gradle/validation/validate-source-patterns.gradle b/gradle/validation/validate-source-patterns.gradle index 7f7d8d990e3..f177c4b273c 100644 --- a/gradle/validation/validate-source-patterns.gradle +++ b/gradle/validation/validate-source-patterns.gradle @@ -3,16 +3,16 @@ // This should be eventually rewritten in plain gradle. For now, delegate to // the ant/groovy script we already have. -configurations { - checkSourceDeps -} - -dependencies { - checkSourceDeps "org.codehaus.groovy:groovy-all:2.4.17" - checkSourceDeps "org.apache.rat:apache-rat:0.11" -} - configure(rootProject) { + configurations { + checkSourceDeps + } + + dependencies { + checkSourceDeps "org.codehaus.groovy:groovy-all:2.4.17" + checkSourceDeps "org.apache.rat:apache-rat" + } + task validateSourcePatterns() { doFirst { ant.taskdef( diff --git a/versions.props b/versions.props index 7f25678af73..94c48ef3752 100644 --- a/versions.props +++ b/versions.props @@ -72,6 +72,7 @@ org.apache.opennlp:opennlp-tools=1.9.1 org.apache.pdfbox:*=2.0.17 org.apache.pdfbox:jempbox=1.8.16 org.apache.poi:*=4.1.1 +org.apache.rat:apache-rat:0.11 org.apache.tika:*=1.23 org.apache.velocity.tools:*=3.0 org.apache.xmlbeans:xmlbeans=3.1.0