lucene/gradle/validation/jar-checks.gradle

// This adds validation of project dependencies:
// 1) license file
// 2) notice file
// 3) checksum validation/ generation.

import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.codec.digest.MessageDigestAlgorithms

// This should be false only for debugging.
def failOnError = true

buildscript {
  repositories {
    mavenCentral()
  }

  dependencies {
    classpath 'commons-codec:commons-codec:1.13'
  }
}

// Configure license checksum folder for top-level projects.
// (The file("licenses") inside the configure scope resolves
// relative to the current project so they're not the same).
configure(project(":lucene")) {
  ext.licensesDir = file("licenses")
}
configure(project(":solr")) {
  ext.licensesDir = file("licenses-gradle")
}

// All known license types. If 'noticeOptional' is true then
// the notice file must accompany the license.
def licenseTypes = [
    "ASL": [name: "Apache Software License 2.0"],
    "BSD": [name: "Berkeley Software Distribution"],
    //BSD like just means someone has taken the BSD license and put in their name, copyright, or it's a very similar license.
    "BSD_LIKE": [name: "BSD like license"],
    "CDDL": [name: "Common Development and Distribution License", noticeOptional: true],
    "CPL": [name: "Common Public License"],
    "EPL": [name: "Eclipse Public License Version 1.0", noticeOptional: true],
    "MIT": [name: "Massachusetts Institute of Tech. License", noticeOptional: true],
    "MPL": [name: "Mozilla Public License", noticeOptional: true /* NOT SURE on the required notice */],
    "PD": [name: "Public Domain", noticeOptional: true],
    "SUN": [name: "Sun Open Source License", noticeOptional: true],
    "COMPOUND": [name: "Compound license (details in NOTICE file)."],
]

allprojects {
  task licenses() {
    group = 'Dependency validation'
    description = "Apply all dependency/ license checks."
  }
  check.dependsOn(licenses)
}

subprojects {
  // Configure jarValidation configuration for all projects. Any dependency
  // declared on this configuration (or any configuration it extends from) will
  // be verified.
  configurations {
    jarValidation
  }

  // For Java projects, add all dependencies from the following configurations
  // to jar validation
  plugins.withType(JavaPlugin) {
    configurations {
      jarValidation {
        extendsFrom runtimeClasspath
        extendsFrom compileClasspath
        extendsFrom testRuntimeClasspath
        extendsFrom testCompileClasspath
      }
    }
  }

  // Collects dependency JAR information for a project and saves it in
  // project.ext.jarInfos. Each dependency has a map of attributes
  // which make it easier to process it later on (name, hash, origin module,
  // see the code below for details).
  task collectJarInfos() {
    dependsOn configurations.jarValidation

    doFirst {
      // We only care about this module's direct dependencies. Anything imported
      // from other modules will be taken care of over there.
      def ownDeps = configurations.detachedConfiguration()
          .extendsFrom(configurations.jarValidation)
          .copyRecursive { dep ->
            !(dep instanceof org.gradle.api.artifacts.ProjectDependency)
          }

      project.ext.jarInfos = ownDeps.resolvedConfiguration.resolvedArtifacts.collect { resolvedArtifact ->
        def file = resolvedArtifact.file
        return [
            name: resolvedArtifact.name,
            jarName: file.toPath().getFileName().toString(),
            path: file,
            module: resolvedArtifact.moduleVersion,
            checksum: new DigestUtils(MessageDigestAlgorithms.SHA_1).digestAsHex(file),
            // We keep count of the files referenced by this dependency (sha, license, notice, etc.)
            // so that we can determine unused files later on.
            referencedFiles: []
        ]
      }
    }
  }

  // Verifies that each JAR has a corresponding checksum and that it matches actual JAR available for this dependency.
  task validateJarChecksums() {
    group = 'Dependency validation'
    description = "Validate checksums of dependencies"
    dependsOn collectJarInfos

    doLast {
      def errors = []
      jarInfos.each { dep ->
        def expectedChecksumFile = file("${licensesDir}/${dep.jarName}.sha1")
        if (!expectedChecksumFile.exists()) {
          errors << "Dependency checksum missing ('${dep.module}'), expected it at: ${expectedChecksumFile}"
        } else {
          dep.referencedFiles += expectedChecksumFile
          def expected = expectedChecksumFile.getText("UTF-8").trim()
          def actual = dep.checksum.trim()
          if (expected.compareToIgnoreCase(actual) != 0) {
            errors << "Dependency checksum mismatch ('${dep.module}'), expected it to be: ${expected}, but was: ${actual}"
          } else {
            logger.log(LogLevel.INFO, "Dependency checksum OK ('${dep.module}')")
          }
        }
      }

      if (errors) {
        def msg = "Dependency checksum validation failed:\n  - " + errors.join("\n  - ")
        if (failOnError) {
          throw new GradleException(msg)
        } else {
          logger.log(LogLevel.WARN, "WARNING: ${msg}")
        }
      }
    }
  }

  // Update dependency checksums
  task updateChecksums() {
    group = 'Dependency validation'
    description = "Write or update checksums of dependencies"
    dependsOn collectJarInfos

    doLast {
      licensesDir.mkdirs()
      jarInfos.each { dep ->
        def expectedChecksumFile = file("${licensesDir}/${dep.jarName}.sha1")
        if (expectedChecksumFile.exists()) {
          def expected = expectedChecksumFile.getText("UTF-8").trim()
          def actual = dep.checksum.trim()
          if (expected.compareToIgnoreCase(actual) == 0) {
            return;
          }
        }

        logger.log(LogLevel.LIFECYCLE, "Updating checksum ('${dep.module}'): ${expectedChecksumFile}")
        expectedChecksumFile.write(dep.checksum.trim(), "UTF-8")
      }
    }
  }

  // Locate the set of license file candidates for this dependency. We
  // search for [jar-or-prefix]-LICENSE-[type].txt
  // where 'jar-or-prefix' can be any '-'-delimited prefix of the dependency JAR's name.
  // So for 'commons-io' it can be 'commons-io-LICENSE-foo.txt' or
  // 'commons-LICENSE.txt'
  task validateJarLicenses() {
    group = 'Dependency validation'
    description = "Validate license and notice files of dependencies"
    dependsOn collectJarInfos

    doLast {
      def errors = []
      jarInfos.each { dep ->
        def baseName = dep.name
        def found = []
        def candidates = []
        while (true) {
          candidates += file("${licensesDir}/${baseName}-LICENSE-[type].txt")
          found += fileTree(dir: licensesDir, include: "${baseName}-LICENSE-*.txt").files
          def prefix = baseName.replaceAll(/[\-][^-]+$/, "")
          if (found || prefix == baseName) {
            break
          }
          baseName = prefix
        }

        if (found.size() == 0) {
          errors << "License file missing ('${dep.module}'), expected it at: ${candidates.join(" or ")}," +
              " where [type] can be any of ${licenseTypes.keySet()}."
        } else if (found.size() > 1) {
          errors << "Multiple license files matching for ('${dep.module}'): ${found.join(", ")}"
        } else {
          def licenseFile = found.get(0)
          dep.referencedFiles += licenseFile
          def m = (licenseFile.name =~ /LICENSE-(.+)\.txt$/)
          if (!m) throw new GradleException("License file name doesn't contain license type?: ${licenseFile.name}")

          def licenseName = m[0][1]
          def licenseType = licenseTypes[licenseName]
          if (!licenseType) {
            errors << "Unknown license type suffix for ('${dep.module}'): ${licenseFile} (must be one of ${licenseTypes.keySet()})"
          } else {
            logger.log(LogLevel.INFO, "Dependency license file OK ('${dep.module}'): " + licenseName)

            // Look for sibling NOTICE file.
            def noticeFile = file(licenseFile.path.replaceAll(/\-LICENSE-.+/, "-NOTICE.txt"))
            if (noticeFile.exists()) {
              dep.referencedFiles += noticeFile
              logger.log(LogLevel.INFO, "Dependency notice file OK ('${dep.module}'): " + noticeFile)
            } else if (!licenseType.noticeOptional) {
              errors << "Notice file missing for ('${dep.module}'), expected it at: ${noticeFile}"
            }
          }
        }
      }

      if (errors) {
        def msg = "Certain license/ notice files are missing:\n  - " + errors.join("\n  - ")
        if (failOnError) {
          throw new GradleException(msg)
        } else {
          logger.log(LogLevel.WARN, "WARNING: ${msg}")
        }
      }
    }
  }

  licenses.dependsOn validateJarChecksums, validateJarLicenses
}

// Check for dangling files in the licenses folder.
configure([project(":solr"), project(":lucene"), ]) {
  def validationTasks = subprojects.collect { it.tasks.matching { it.name == "licenses" } }

  task checkDanglingLicenseFiles() {
    dependsOn validationTasks

    ext {
      exclude = []
    }

    doFirst {
      def allReferenced = validationTasks.collectMany { task ->
          task.project.jarInfos.collectMany { it.referencedFiles }
        }
        .collect { it.toString() }

      def patterns = ext.exclude
      def allExisting = fileTree(licensesDir, {
        exclude patterns
      }).files.collect { it.toString() }

      def dangling = (allExisting - allReferenced).sort()

      if (dangling) {
        gradle.buildFinished {
          logger.warn("WARNING: there were unreferenced files under license folder:\n  - ${dangling.join("\n  - ")}")
        }
      }
    }
  }

  licenses.dependsOn checkDanglingLicenseFiles
}

// Exclude files that are not a result of direct dependencies but have to be there.
// It would be probably better to move non-dependency licenses into the actual project
// where they're used and only assemble them for the distribution package.
configure(project(":lucene")) {
  checkDanglingLicenseFiles {
    exclude += [
        "elegant-icon-font-*",
        "ant-*",
        "ivy-*",
    ]
  }
}

configure(project(":solr")) {
  checkDanglingLicenseFiles {
    exclude += [
        "README.committers.txt",
    ]
  }
}

// Disable validation for these projects (should it be disabled?)
configure(project(":solr:solr-ref-guide")) {
  [validateJarLicenses, validateJarChecksums, updateChecksums].each { task ->
    task.enabled = false
  }
}