lucene/gradle/generation/regenerate.gradle

import groovy.json.JsonOutput
import groovy.json.JsonSlurper
import java.util.function.Function

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Create common 'regenerate' task sub-tasks can hook into.

/**
 * Compute all "checksummed" key-value pairs.
 */
def computeChecksummedEntries = { Task sourceTask ->
  // An flat ordered map of key-value pairs.
  Map<String, String> allEntries = new TreeMap<>()

  // Make sure all input properties are either simple strings
  // or closures returning simple strings.
  //
  // Don't overcomplicate things with other serializable types.
  Map<String, Object> props = sourceTask.inputs.properties
  props.forEach { key, val ->
    // Handle closures and other lazy providers.
    if (val instanceof Provider<?>) {
      val = val.get()
    }
    if (val instanceof Closure<?>) {
      val = val.call()
    }

    if (!(val instanceof String)) {
      throw new GradleException("Input properties of wrapped tasks must all be " +
          "strings: ${key} in ${sourceTask.name} is not.")
    }
    allEntries.put("property:" + key, (String) val)
  }

  // Collect all of task inputs/ output files and compute their checksums.
  FileCollection allFiles = sourceTask.inputs.files + sourceTask.outputs.files

  // Compute checksums for root-project relative paths
  allFiles.files.forEach { file ->
    allEntries.put(
        sourceTask.project.rootDir.relativePath(file),
        file.exists() ? buildinfra.sha1Digest().digestAsHex(file).trim() : "--")
  }

  return allEntries
}

configure(rootProject) {
  ext {
    /**
     * Utility function to read a file, apply changes to its content and write it back.
     */
    modifyFile = { File path, Function<String, String> modify ->
      Function<String, String> normalizeEols = { text -> text.replace("\r\n", "\n") }

      String original = path.getText("UTF-8")
      String modified = normalizeEols.apply(original)
      modified = modify.apply(modified)
      modified = normalizeEols.apply(modified)
      if (!original.equals(modified)) {
        path.write(modified, "UTF-8")
      }
    }
  }
}

configure([
        project(":lucene:analysis:common"),
        project(":lucene:analysis:icu"),
        project(":lucene:analysis:kuromoji"),
        project(":lucene:analysis:nori"),
        project(":lucene:backward-codecs"),
        project(":lucene:core"),
        project(":lucene:queryparser"),
        project(":lucene:expressions"),
        project(":lucene:test-framework"),
]) {
    task regenerate() {
      description "Rerun any code or static data generation tasks."
      group "generation"
    }

    project.ext {
      // This utility method implements the logic required for "persistent" incremental
      // source-generating tasks. The idea is simple, the implementation quite complex.
      //
      // The idea is that, given source-generating task "sourceTaskInternal" (note the suffix),
      // we create a bunch of other tasks that perform checksum generation, validation and
      // source task skipping. For example, let's say we have a task 'genFooInternal";
      // the following tasks would be created
      //
      // genFooChecksumLoad
      // genFooChecksumSave
      // genFooChecksumCheck (fails if checksums are inconsistent)
      //
      // and the following set of dependencies would be created (with additional
      // constraints to run them in this particular order!).
      //
      // genFoo.dependsOn [genFooChecksumLoad, genFooInternal, genFooChecksumSave]
      //
      // Checksums are persisted and computed from sourceTask's inputs/outputs. If the
      // persisted checksums are identical to the now-current checksums, the "internal" task
      // is skipped (using onlyIf { false }).
      //
      // Implementation-wise things get complicated because gradle doesn't have the notion
      // of "ordered" task execution with respect to task AND its dependencies (we can add
      // constraints to each node in the execution graph but not node-and-dependencies).
      //
      // sourceTask - the task to wrap
      // extraConfig - a map with extra (optional) configuration options.
      //   andThenTasks: any other tasks that should be scheduled to run after the internal task and
      //     before checksum calculation/ saving. spotless is a good example of this.
      //   ignoreWithSource: any other tasks that should be ignored if the internal task is ignored
      //     (checksums are identical)
      //   mustRunBefore: any tasks which should be scheduled to run after the internal task.
      wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
        if (!sourceTask.name.endsWith("Internal")) {
          throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}")
        }

        String sourceTaskName = sourceTask.name.replaceAll('Internal$', '')

        def toList = { value ->
          if (value instanceof List) {
            return value
          } else if (value == null) {
            return []
          } else {
            return [ value ]
          }
        }

        List<Object> andThenTasks = toList(extraConfig.get("andThenTasks"))
        List<Object> ignoreWithSource =  toList(extraConfig.get("ignoreWithSource"))

        // Schedule must-run-afters
        List<Object> mustRunBefore = toList(extraConfig.get("mustRunBefore"))
        // TODO: maybe ensure all task refs here are strings?
        tasks.matching { it.name in mustRunBefore }.configureEach {
          mustRunAfter sourceTask
        }

        // Create checksum-loader task.
        Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", {
          ext {
            checksumMatch = true
          }

          doFirst {
            // Current persisted task input/outputs (file checksums, properties)
            ext.currentChecksums = computeChecksummedEntries(sourceTask)

            // Load any previously written checksums
            ext.savedChecksums = new TreeMap<>()
            ext.checksumsFile = project.file("src/generated/checksums/${sourceTaskName}.json")
            if (checksumsFile.exists()) {
              savedChecksums.putAll(new JsonSlurper().parse(checksumsFile) as Map)
            }

            // Compare saved and current checksums for subsequent tasks.
            ext.checksumMatch = (savedChecksums.equals(currentChecksums))
          }
        })

        Task checksumCheckTask = tasks.create("${sourceTaskName}ChecksumCheck", {
          dependsOn checksumLoadTask

          doFirst {
            if (!checksumLoadTask.checksumMatch) {
              // This can be made prettier but leave it verbose for now:
              Map<String, String> current = checksumLoadTask.currentChecksums
              Map<String, String> expected = checksumLoadTask.savedChecksums

              def same = current.intersect(expected)
              current = current - same
              expected = expected - same

              throw new GradleException("Checksums mismatch for derived resources; you might have" +
                  " modified a generated resource (regenerate task: ${sourceTaskName}):\n" +
                  "Current:\n  ${current.entrySet().join('\n  ')}\n\n" +
                  "Expected:\n  ${expected.entrySet().join('\n  ')}\n\n" +
                  "Input files for this task are:\n  " + sourceTask.inputs.files.join('\n  ') + "\n\n" +
                  "Files generated by this task are:\n  " + sourceTask.outputs.files.join('\n  ')
              )
            }
          }
        })
        check.dependsOn checksumCheckTask

        Task checksumSaveTask = tasks.create("${sourceTaskName}ChecksumSave", {
          dependsOn checksumLoadTask

          doFirst {
            File checksumsFile = checksumLoadTask.ext.checksumsFile
            checksumsFile.parentFile.mkdirs()

            // Recompute checksums after the task has completed and write them.
            def updatedChecksums = computeChecksummedEntries(sourceTask)
            checksumsFile.setText(
                JsonOutput.prettyPrint(JsonOutput.toJson(new TreeMap<String, String>(updatedChecksums))), "UTF-8")

            logger.warn("Updated generated file checksums for task ${sourceTask.path}.")
          }
        })

        Task conditionalTask = tasks.create("${sourceTaskName}", {
          def deps = [
              checksumLoadTask,
              sourceTask,
              *andThenTasks,
              checksumSaveTask
          ].flatten()

          dependsOn deps
          mustRunInOrder deps

          doFirst {
            if (checksumLoadTask.checksumMatch && !sourceTask.didWork) {
              logger.lifecycle("Checksums consistent with sources, skipping task: ${sourceTask.path}")
            }
          }
        })

        // Load checksums before the source task executes, otherwise it's always ignored.
        project.afterEvaluate {
          resolveTaskRefs([sourceTask, *ignoreWithSource]).each { t ->
            t.configure {
              dependsOn checksumLoadTask
            }
          }
        }

        // Copy the description and group from the source task.
        project.afterEvaluate {
          conditionalTask.group sourceTask.group
          conditionalTask.description sourceTask.description + " (if sources changed)"

          // Hide low-level tasks from help.
          sourceTask.group = null
          sourceTask.description sourceTask.description + " (low-level)"
        }

        // Set conditional execution only if checksum mismatch occurred.
        if (!gradle.startParameter.isRerunTasks()) {
          project.afterEvaluate {
            resolveTaskRefs([sourceTask, *ignoreWithSource, checksumSaveTask]).each { t ->
              t.configure {
                logger.info("Making " + t.name + " run only if " + checksumLoadTask.name + " indicates changes")
                onlyIf { !checksumLoadTask.checksumMatch }
              }
            }
          }
        }

        return conditionalTask
      }
    }
}