LUCENE-9900: Regenerate/ run ICU only if inputs changed (#61)

gradle/generation/icu.gradle

@@ -43,12 +43,21 @@ configure(project(":lucene:analysis:icu")) {
     icupkg = file("${icuBinDir}/icupkg")
   }
 
+  def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
+
   task genUtr30DataFiles() {
-    dependsOn Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
+    dependsOn icuCompileTask
 
     // May be undefined yet, so use a provider.
     dependsOn { sourceSets.tools.runtimeClasspath }
 
+    // gennorm generates file order-dependent output, so make it constant here.
+    def inputFiles = fileTree(dir: utr30DataDir, include: "*.txt").asList().toSorted(Comparator.comparing { File f -> f.name })
+    def outputFile = file("src/resources/org/apache/lucene/analysis/icu/utr30.nrm")
+
+    inputs.files inputFiles
+    outputs.file outputFile
+
     doFirst {
       // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm
       def v = getVersion('com.ibm.icu', 'icu4j');
@@ -71,9 +80,7 @@ configure(project(":lucene:analysis:icu")) {
             utr30DataDir,
             "-o",
             "${buildDir}/utr30.tmp",
-            "nfc.txt", "nfkc.txt", "nfkc_cf.txt", "BasicFoldings.txt",
-            "DiacriticFolding.txt", "DingbatFolding.txt", "HanRadicalFolding.txt",
-            "NativeDigitFolding.txt"
+            *(inputFiles.collect { it.name })
         ]
       }
 
@@ -82,7 +89,7 @@ configure(project(":lucene:analysis:icu")) {
         args = [
             "-tb",
             "${buildDir}/utr30.tmp",
-            "src/resources/org/apache/lucene/analysis/icu/utr30.nrm"
+            outputFile
         ]
       }
     }
@@ -92,6 +99,12 @@ configure(project(":lucene:analysis:icu")) {
     // May be undefined yet, so use a provider.
     dependsOn { sourceSets.tools.runtimeClasspath }
 
+    def sourceDir = file("src/data/uax29")
+    def targetDir = file("src/resources/org/apache/lucene/analysis/icu/segmentation")
+
+    inputs.files fileTree(dir: sourceDir, include: "*.rbbi")
+    outputs.files fileTree(dir: targetDir, include: "*.brk")
+
     doFirst {
       project.javaexec {
         main = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"
@@ -99,15 +112,13 @@ configure(project(":lucene:analysis:icu")) {
 
         ignoreExitValue false
         enableAssertions true
-        args = [
-            "src/data/uax29",
-            "src/resources/org/apache/lucene/analysis/icu/segmentation"
-        ]
+        args = [ sourceDir, targetDir ]
       }
     }
   }
 
-  regenerate.dependsOn genUtr30DataFiles, genRbbi
+  regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFiles, [ ignoreWithSource: icuCompileTask ])
+  regenerate.dependsOn wrapWithPersistentChecksums(genRbbi)
 
   task compileIcuWindows() {
     doFirst {

gradle/generation/javacc.gradle

@@ -252,9 +252,9 @@ configure(project(":lucene:queryparser")) {
     description "Regenerate query parsers (javacc syntax definitions)."
     group "generation"
 
-    dependsOn wrapWithPersistentChecksums(javaccParserClassic, "spotlessApply"),
-        wrapWithPersistentChecksums(javaccParserSurround, "spotlessApply"),
-        wrapWithPersistentChecksums(javaccParserFlexible, "spotlessApply")
+    dependsOn wrapWithPersistentChecksums(javaccParserClassic, [ andThenTasks: "spotlessApply" ]),
+        wrapWithPersistentChecksums(javaccParserSurround, [ andThenTasks: "spotlessApply" ]),
+        wrapWithPersistentChecksums(javaccParserFlexible, [ andThenTasks: "spotlessApply" ])
   }
 
   regenerate.dependsOn javacc

gradle/generation/jflex.gradle

@@ -49,7 +49,7 @@ configure(project(":lucene:core")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateStandardTokenizer, "spotlessApply")
+  regenerate.dependsOn wrapWithPersistentChecksums(generateStandardTokenizer, [ andThenTasks: "spotlessApply" ])
 }
 
 configure(project(":lucene:analysis:common")) {
@@ -118,10 +118,10 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizer, "spotlessApply"),
-      wrapWithPersistentChecksums(generateClassicTokenizer, "spotlessApply"),
-      wrapWithPersistentChecksums(generateUAX29URLEmailTokenizer, "spotlessApply"),
-      wrapWithPersistentChecksums(generateHTMLStripCharFilter, "spotlessApply")
+  regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizer, [ andThenTasks: "spotlessApply" ]),
+      wrapWithPersistentChecksums(generateClassicTokenizer, [ andThenTasks: "spotlessApply" ]),
+      wrapWithPersistentChecksums(generateUAX29URLEmailTokenizer, [ andThenTasks: "spotlessApply" ]),
+      wrapWithPersistentChecksums(generateHTMLStripCharFilter, [ andThenTasks: "spotlessApply" ])
 }
 
 class JFlexTask extends DefaultTask {

gradle/generation/moman.gradle

@@ -99,8 +99,8 @@ configure(project(":lucene:core")) {
     description "Regenerate Moman-based sources."
     group "generation"
 
-    dependsOn wrapWithPersistentChecksums(utilGenPacked, "spotlessApply")
-    dependsOn wrapWithPersistentChecksums(utilGenLev, "spotlessApply")
+    dependsOn wrapWithPersistentChecksums(utilGenPacked, [ andThenTasks: "spotlessApply" ])
+    dependsOn wrapWithPersistentChecksums(utilGenLev, [ andThenTasks: "spotlessApply" ])
   }
 
   regenerate.dependsOn moman

gradle/generation/regenerate.gradle

@@ -56,9 +56,23 @@ configure([
       // constraints to each node in the execution graph but not node-and-dependencies).
       //
       // sourceTask - the task to wrap
-      // otherTasks - other tasks that should be scheduled to run after source task and
-      //   before checksum calculation.
-      wrapWithPersistentChecksums = { Task sourceTask, Object... otherTasks ->
+      // extraConfig - a map with extra (optional) configuration options.
+      //   andThenTasks: other tasks that should be scheduled to run after source task and
+      //     before checksum calculation.
+      wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
+        def toList = { value ->
+          if (value instanceof List) {
+            return value
+          } else if (value == null) {
+            return []
+          } else {
+            return [ value ]
+          }
+        }
+
+        List<Object> andThenTasks = toList(extraConfig.get("andThenTasks"))
+        List<Object> ignoreWithSource =  toList(extraConfig.get("ignoreWithSource"))
+
         // Create checksum-loader task.
         Task checksumLoadTask = tasks.create("${sourceTask.name}ChecksumLoad", {
           ext {
@@ -140,7 +154,7 @@ configure([
           def deps = [
               checksumLoadTask,
               sourceTask,
-              *otherTasks,
+              *andThenTasks,
               checksumSaveTask
           ].flatten()
 
@@ -163,7 +177,7 @@ configure([
         // Set conditional execution only if checksum mismatch occurred.
         if (!gradle.startParameter.isRerunTasks()) {
           project.afterEvaluate {
-            resolveTaskRefs([sourceTask, checksumSaveTask]).each { t ->
+            resolveTaskRefs([sourceTask, *ignoreWithSource, checksumSaveTask]).each { t ->
               t.configure {
                 logger.info("Making " + t.name + " run only if " + checksumLoadTask.name + " indicates changes")
                 onlyIf { !checksumLoadTask.checksumMatch }

lucene/analysis/icu/src/generated/checksums/genRbbi.json

@@ -0,0 +1,6 @@
+{
+    "lucene/analysis/icu/src/data/uax29/Default.rbbi": "71bfaee5e81ac272aff828d1e44d0612be1b8363",
+    "lucene/analysis/icu/src/data/uax29/MyanmarSyllable.rbbi": "4c6817658b454add5ec1f9ac8c0015ce8eb3b5f2",
+    "lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk": "1b9013b7ef4ba32a851a330c58a8fa820b9dda79",
+    "lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk": "cc023ec17e0148518086098691785a32b88ee09a"
+}

lucene/analysis/icu/src/generated/checksums/genUtr30DataFiles.json

@@ -0,0 +1,11 @@
+{
+    "lucene/analysis/icu/src/data/utr30/BasicFoldings.txt": "fc5badae307c5b740f8dd81407e4be04e3e2fde6",
+    "lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt": "2d48c24758ed9322b49eee84c23c74b7f11337c6",
+    "lucene/analysis/icu/src/data/utr30/DingbatFolding.txt": "ca562b2029e208bdf07cb2e5f689e509a336265e",
+    "lucene/analysis/icu/src/data/utr30/HanRadicalFolding.txt": "f5848aa38cbf1a11131d59384f31477f3b7f077b",
+    "lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt": "434ad7eb17fc32a441385795e4a4aa7e589ec30a",
+    "lucene/analysis/icu/src/data/utr30/nfc.txt": "ec95d7f7e5910791717234dd09efc4b13dc32d35",
+    "lucene/analysis/icu/src/data/utr30/nfkc.txt": "b8e91bc64e354af505d51f6072a43c90c6b4d1b5",
+    "lucene/analysis/icu/src/data/utr30/nfkc_cf.txt": "22d90ea4a7771e6ddebaaeb9438e98ce625e16f5",
+    "lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm": "32a6a3198039883c93f9ebef31fe24c1029f2b07"
+}