lucene/gradle/validation/rat-sources.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import groovy.xml.NamespaceBuilder

// Configure rat dependencies for use in the custom task.
configure(rootProject) {
    configurations {
        ratDeps
    }

    dependencies {
        ratDeps "org.apache.rat:apache-rat:${scriptDepVersions['apache-rat']}"
    }
}

// Configure the rat validation task and all scanned directories.
allprojects {
    task("rat", type: RatTask) {
        group = 'Verification'
        description = 'Runs Apache Rat checks.'

        def defaultScanFileTree = project.fileTree(projectDir, {
            // Don't check under the project's build folder.
            exclude project.buildDir.name

            // Exclude any generated stuff.
            exclude "src/generated"

            // Don't check any of the subprojects - they have their own rat tasks.
            exclude subprojects.collect { it.projectDir.name }

            // At the module scope we only check selected file patterns as folks have various .gitignore-d resources
            // generated by IDEs, etc.
            include "**/*.gradle"
            include "**/*.xml"
            include "**/*.md"
            include "**/*.py"
            include "**/*.sh"
            include "**/*.bat"

            // Include selected patterns from any source folders. We could make this
            // relative to source sets but it seems to be of little value - all our source sets
            // live under 'src' anyway.
            include "src/**"
            exclude "src/**/*.png"
            exclude "src/**/*.txt"
            exclude "src/**/*.zip"
            exclude "src/**/*.properties"
            exclude "src/**/*.utf8"

            // Conditionally apply module-specific patterns. We do it here instead
            // of reconfiguring each project because the provider can be made lazy
            // and it's easier to manage this way.
            switch (project.path) {
                case ":":
                    include "gradlew"
                    include "gradlew.bat"
                    exclude ".gradle"
                    exclude ".idea"
                    exclude ".muse"
                    exclude ".git"

                    // Exclude github stuff (templates, workflows).
                    exclude ".github"

                    // The root project also includes patterns for the boostrap (buildSrc) and composite
                    // projects. Include their sources in the scan.
                    include "buildSrc/src/**"
                    include "dev-tools/missing-doclet/src/**"

                    // do not let RAT attempt to scan a python venv, it gets lost and confused...
                    exclude "dev-tools/aws-jmh/build/**"
                    break

                case ":lucene:analysis:morfologik":
                    exclude "src/**/*.info"
                    exclude "src/**/*.input"
                    break

                case ":lucene:analysis:opennlp":
                    exclude "src/**/en-test-lemmas.dict"
                    break

                case ":lucene:demo":
                    exclude "src/**/knn-token-vectors"
                    break

                case ":lucene:test-framework":
                    exclude "src/**/europarl.lines.txt.seek"
                    break

                case ":lucene:analysis:common":
                case ":lucene:analysis.tests":
                    exclude "src/**/*.aff"
                    exclude "src/**/*.dic"
                    exclude "src/**/*.good"
                    exclude "src/**/*.sug"
                    exclude "src/**/*.wrong"
                    exclude "src/**/charfilter/*.htm*"
                    exclude "src/**/*LuceneResourcesWikiPage.html"
                    exclude "src/**/*.rslp"
                    break

                case ":lucene:benchmark":
                    exclude "data/"
                    break
            }
        })
        inputFileTrees.add(defaultScanFileTree)
    }
}

/**
 * An Apache RAT adapter that validates whether files contain acceptable licenses.
 */
@CacheableTask
class RatTask extends DefaultTask {

    @InputFiles
    @PathSensitive(PathSensitivity.RELATIVE)
    @IgnoreEmptyDirectories
    final ListProperty<ConfigurableFileTree> inputFileTrees = project.objects.listProperty(ConfigurableFileTree)

    @OutputFile
    final RegularFileProperty xmlReport = project.objects.fileProperty().convention(
        project.layout.buildDirectory.file("rat/rat-report.xml"))

    def generateReport(File reportFile) {
        // Set up ant rat task.
        def ratClasspath = project.rootProject.configurations.ratDeps.asPath
        ant.setLifecycleLogLevel(AntBuilder.AntMessagePriority.ERROR)
        ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', classpath: ratClasspath)

        // Collect all output files for debugging.
        String inputFileList = inputFileTrees.get().collectMany { fileTree ->
            fileTree.asList()
        }.sort().join("\n")
        project.file(reportFile.path.replaceAll('.xml$', '-filelist.txt')).setText(inputFileList, "UTF-8")

        // Run rat via ant.
        ant.report(format: 'xml', reportFile: reportFile, addDefaultLicenseMatchers: true) {
            // Pass all gradle file trees to the ant task (Gradle's internal adapters are used).
            inputFileTrees.get().each { fileTree ->
                fileTree.addToAntBuilder(ant, 'resources', FileCollection.AntType.ResourceCollection)
            }

            // BSD 4-clause stuff (is disallowed below)
            substringMatcher(licenseFamilyCategory: "BSD4 ", licenseFamilyName: "Original BSD License (with advertising clause)") {
                pattern(substring: "All advertising materials")
            }

            // BSD-like stuff
            substringMatcher(licenseFamilyCategory: "BSD  ", licenseFamilyName: "Modified BSD License") {
                // brics automaton
                pattern(substring: "Copyright (c) 2001-2009 Anders Moeller")
                // snowball
                pattern(substring: "Copyright (c) 2001, Dr Martin Porter")
                // UMASS kstem
                pattern(substring: "THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS")
                // Egothor
                pattern(substring: "Egothor Software License version 1.00")
                // JaSpell
                pattern(substring: "Copyright (c) 2005 Bruno Martins")
                // d3.js
                pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS")
                // highlight.js
                pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS")
            }

            // MIT-like
            substringMatcher(licenseFamilyCategory: "MIT  ", licenseFamilyName:"Modified BSD License") {
                // ICU license
                pattern(substring: "Permission is hereby granted, free of charge, to any person obtaining a copy")
                // ui-grid
                pattern(substring: " ; License: MIT")
            }

            // Apache
            substringMatcher(licenseFamilyCategory: "AL   ", licenseFamilyName: "Apache") {
                pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
                // this is the old - school one under some files
                pattern(substring: 'Licensed under the Apache License, Version 2.0 (the "License")')
            }

            substringMatcher(licenseFamilyCategory: "GEN  ", licenseFamilyName: "Generated") {
                // svg files generated by gnuplot
                pattern(substring: "Produced by GNUPLOT")
                // snowball stemmers generated by snowball compiler
                pattern(substring: "Generated by Snowball")
                // parsers generated by antlr
                pattern(substring: "ANTLR GENERATED CODE")
            }

            approvedLicense(familyName: "Apache")
            approvedLicense(familyName: "The MIT License")
            approvedLicense(familyName: "Modified BSD License")
            approvedLicense(familyName: "Generated")
        }
    }

    def printUnknownFiles(File reportFile) {
        def ratXml = new XmlParser().parse(reportFile)
        def errors = []
        ratXml.resource.each { resource ->
            if (resource.'license-approval'.@name[0] == "false") {
                errors << "Unknown license: ${resource.@name}"
            }
        }
        if (errors) {
            throw new GradleException("Found " + errors.size() + " file(s) with errors:\n" +
                errors.collect{ msg -> "  - ${msg}" }.join("\n"))
        }
    }

    @TaskAction
    def execute() {
        def origEncoding = System.getProperty("file.encoding")
        try {
            File reportFile = xmlReport.get().asFile
            generateReport(reportFile)
            printUnknownFiles(reportFile)
        } finally {
            if (System.getProperty("file.encoding") != origEncoding) {
                throw new GradleException("Something is wrong: Apache RAT changed file.encoding to ${System.getProperty('file.encoding')}?")
            }
        }
    }
}