import groovy.json.JsonOutput import groovy.json.JsonSlurper import java.util.function.Function /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Create common 'regenerate' task sub-tasks can hook into. /** * Compute all "checksummed" key-value pairs. */ def computeChecksummedEntries = { Task sourceTask -> // An flat ordered map of key-value pairs. Map allEntries = new TreeMap<>() // Make sure all input properties are either simple strings // or closures returning simple strings. // // Don't overcomplicate things with other serializable types. Map props = sourceTask.inputs.properties props.forEach { key, val -> // Handle closures and other lazy providers. if (val instanceof Provider) { val = val.get() } if (val instanceof Closure) { val = val.call() } if (!(val instanceof String)) { throw new GradleException("Input properties of wrapped tasks must all be " + "strings: ${key} in ${sourceTask.name} is not.") } allEntries.put("property:" + key, (String) val) } // Collect all of task inputs/ output files and compute their checksums. FileCollection allFiles = sourceTask.inputs.files + sourceTask.outputs.files // Compute checksums for root-project relative paths allFiles.files.forEach { file -> allEntries.put( sourceTask.project.rootDir.relativePath(file), file.exists() ? buildinfra.sha1Digest().digestAsHex(file).trim() : "--") } return allEntries } configure(rootProject) { ext { /** * Utility function to read a file, apply changes to its content and write it back. */ modifyFile = { File path, Function modify -> Function normalizeEols = { text -> text.replace("\r\n", "\n") } String original = path.getText("UTF-8") String modified = normalizeEols.apply(original) modified = modify.apply(modified) modified = normalizeEols.apply(modified) if (!original.equals(modified)) { path.write(modified, "UTF-8") } } } } configure([ project(":lucene:analysis:common"), project(":lucene:analysis:icu"), project(":lucene:analysis:kuromoji"), project(":lucene:analysis:nori"), project(":lucene:backward-codecs"), project(":lucene:core"), project(":lucene:queryparser"), project(":lucene:expressions"), project(":lucene:test-framework"), ]) { task regenerate() { description "Rerun any code or static data generation tasks." group "generation" } project.ext { // This utility method implements the logic required for "persistent" incremental // source-generating tasks. The idea is simple, the implementation quite complex. // // The idea is that, given source-generating task "sourceTaskInternal" (note the suffix), // we create a bunch of other tasks that perform checksum generation, validation and // source task skipping. For example, let's say we have a task 'genFooInternal"; // the following tasks would be created // // genFooChecksumLoad // genFooChecksumSave // genFooChecksumCheck (fails if checksums are inconsistent) // // and the following set of dependencies would be created (with additional // constraints to run them in this particular order!). // // genFoo.dependsOn [genFooChecksumLoad, genFooInternal, genFooChecksumSave] // // Checksums are persisted and computed from sourceTask's inputs/outputs. If the // persisted checksums are identical to the now-current checksums, the "internal" task // is skipped (using onlyIf { false }). // // Implementation-wise things get complicated because gradle doesn't have the notion // of "ordered" task execution with respect to task AND its dependencies (we can add // constraints to each node in the execution graph but not node-and-dependencies). // // sourceTask - the task to wrap // extraConfig - a map with extra (optional) configuration options. // andThenTasks: any other tasks that should be scheduled to run after the internal task and // before checksum calculation/ saving. spotless is a good example of this. // ignoreWithSource: any other tasks that should be ignored if the internal task is ignored // (checksums are identical) // mustRunBefore: any tasks which should be scheduled to run after the internal task. wrapWithPersistentChecksums = { Task sourceTask, Map extraConfig = [:] -> if (!sourceTask.name.endsWith("Internal")) { throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}") } String sourceTaskName = sourceTask.name.replaceAll('Internal$', '') def toList = { value -> if (value instanceof List) { return value } else if (value == null) { return [] } else { return [ value ] } } List andThenTasks = toList(extraConfig.get("andThenTasks")) List ignoreWithSource = toList(extraConfig.get("ignoreWithSource")) // Schedule must-run-afters List mustRunBefore = toList(extraConfig.get("mustRunBefore")) // TODO: maybe ensure all task refs here are strings? tasks.matching { it.name in mustRunBefore }.configureEach { mustRunAfter sourceTask } // Create checksum-loader task. Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", { ext { checksumMatch = true } doFirst { // Current persisted task input/outputs (file checksums, properties) ext.currentChecksums = computeChecksummedEntries(sourceTask) // Load any previously written checksums ext.savedChecksums = new TreeMap<>() ext.checksumsFile = project.file("src/generated/checksums/${sourceTaskName}.json") if (checksumsFile.exists()) { savedChecksums.putAll(new JsonSlurper().parse(checksumsFile) as Map) } // Compare saved and current checksums for subsequent tasks. ext.checksumMatch = (savedChecksums.equals(currentChecksums)) } }) Task checksumCheckTask = tasks.create("${sourceTaskName}ChecksumCheck", { dependsOn checksumLoadTask doFirst { if (!checksumLoadTask.checksumMatch) { // This can be made prettier but leave it verbose for now: Map current = checksumLoadTask.currentChecksums Map expected = checksumLoadTask.savedChecksums def same = current.intersect(expected) current = current - same expected = expected - same throw new GradleException("Checksums mismatch for derived resources; you might have" + " modified a generated resource (regenerate task: ${sourceTaskName}):\n" + "Current:\n ${current.entrySet().join('\n ')}\n\n" + "Expected:\n ${expected.entrySet().join('\n ')}\n\n" + "Input files for this task are:\n " + sourceTask.inputs.files.join('\n ') + "\n\n" + "Files generated by this task are:\n " + sourceTask.outputs.files.join('\n ') ) } } }) check.dependsOn checksumCheckTask Task checksumSaveTask = tasks.create("${sourceTaskName}ChecksumSave", { dependsOn checksumLoadTask doFirst { File checksumsFile = checksumLoadTask.ext.checksumsFile checksumsFile.parentFile.mkdirs() // Recompute checksums after the task has completed and write them. def updatedChecksums = computeChecksummedEntries(sourceTask) checksumsFile.setText( JsonOutput.prettyPrint(JsonOutput.toJson(new TreeMap(updatedChecksums))), "UTF-8") logger.warn("Updated generated file checksums for task ${sourceTask.path}.") } }) Task conditionalTask = tasks.create("${sourceTaskName}", { def deps = [ checksumLoadTask, sourceTask, *andThenTasks, checksumSaveTask ].flatten() dependsOn deps mustRunInOrder deps doFirst { if (checksumLoadTask.checksumMatch && !sourceTask.didWork) { logger.lifecycle("Checksums consistent with sources, skipping task: ${sourceTask.path}") } } }) // Load checksums before the source task executes, otherwise it's always ignored. project.afterEvaluate { resolveTaskRefs([sourceTask, *ignoreWithSource]).each { t -> t.configure { dependsOn checksumLoadTask } } } // Copy the description and group from the source task. project.afterEvaluate { conditionalTask.group sourceTask.group conditionalTask.description sourceTask.description + " (if sources changed)" // Hide low-level tasks from help. sourceTask.group = null sourceTask.description sourceTask.description + " (low-level)" } // Set conditional execution only if checksum mismatch occurred. if (!gradle.startParameter.isRerunTasks()) { project.afterEvaluate { resolveTaskRefs([sourceTask, *ignoreWithSource, checksumSaveTask]).each { t -> t.configure { logger.info("Making " + t.name + " run only if " + checksumLoadTask.name + " indicates changes") onlyIf { !checksumLoadTask.checksumMatch } } } } } return conditionalTask } } }