LUCENE-9862: cleanup of all regenerate tasks. Leaving interim commits for reference.

build.gradle

@@ -148,6 +148,7 @@ apply from: file('gradle/validation/check-broken-links.gradle')
 apply from: file('gradle/validation/spotless.gradle')
 
 // Source or data regeneration tasks
+apply from: file('gradle/generation/regenerate.gradle')
 apply from: file('gradle/generation/jflex.gradle')
 apply from: file('gradle/generation/util.gradle')
 apply from: file('gradle/generation/snowball.gradle')

gradle/generation/icu.gradle

@@ -108,13 +108,7 @@ configure(project(":lucene:analysis:icu")) {
     }
   }
 
-  task regenerate() {
-    description "Regenerate ICU data files"
-    group "generation"
-
-    dependsOn genUtr30DataFiles
-    dependsOn genRbbi
-  }
+  regenerate.dependsOn genUtr30DataFiles, genRbbi
 
   task compileIcuWindows() {
     doFirst {

gradle/generation/javacc.gradle

@@ -242,22 +242,6 @@ configure(project(":lucene:queryparser")) {
     }
   }
 
-  task regenerate() {
-    description "Regenerate any generated sources"
-    group "generation"
-
-    // Run regeneration tasks.
-    dependsOn javaccParserClassic, javaccParserSurround, javaccParserFlexible
-
-    // Clean up and reformat the generated sources after generation.
-    dependsOn "tidy"
-  }
-
-  // Make sure tidy runs after generation, if they're defined.
-  tasks.matching { it.name == "tidy" }.configureEach {
-    mustRunAfter javaccParserClassic, javaccParserSurround, javaccParserFlexible
-  }
-
   task javacc() {
     description "Regenerate query parsers (javacc syntax definitions)."
     group "generation"
@@ -266,6 +250,8 @@ configure(project(":lucene:queryparser")) {
     dependsOn javaccParserSurround
     dependsOn javaccParserFlexible
   }
+
+  regenerate.dependsOn javacc, tidy
 }
 
 // We always regenerate, no need to declare outputs.

gradle/generation/jflex.gradle

@@ -103,6 +103,8 @@ configure(project(":lucene:core")) {
       )
     }
   }
+
+  regenerate.dependsOn jflexStandardTokenizerImpl, "tidy"
 }
 
 configure(project(":lucene:analysis:common")) {
@@ -132,7 +134,7 @@ configure(project(":lucene:analysis:common")) {
     heapSize = "12g"
 
     doFirst {
-      logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires tons of memory).")
+      logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires ${heapSize} of memory!).")
     }
 
     doLast {
@@ -175,25 +177,9 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  task regenerate() {
-    description "Regenerate any generated sources"
-    group "generation"
-
-    // Run regeneration tasks.
-    dependsOn jflexUAX29URLEmailTokenizerImpl
-    dependsOn jflexHTMLStripCharFilter
-    dependsOn jflexClassicTokenizerImpl
-    dependsOn jflexWikipediaTokenizerImpl
-
-    // Clean up and reformat the generated sources after generation.
-    dependsOn "tidy"
-  }
-
-  // Make sure tidy runs after generation, if they're defined.
-  tasks.matching { it.name == "tidy" }.configureEach {
-    mustRunAfter jflexUAX29URLEmailTokenizerImpl,
-        jflexHTMLStripCharFilter,
-        jflexClassicTokenizerImpl,
-        jflexWikipediaTokenizerImpl
-  }
+  regenerate.dependsOn jflexUAX29URLEmailTokenizerImpl,
+          jflexHTMLStripCharFilter,
+          jflexClassicTokenizerImpl,
+          jflexWikipediaTokenizerImpl,
+          "tidy"
 }

gradle/generation/kuromoji.gradle

@@ -128,5 +128,8 @@ configure(project(":lucene:analysis:kuromoji")) {
         })
       }
     }
+
+    // TODO: should we include this in default regeneration?
+    // regenerate.dependsOn compileMecab
   }
 }

gradle/generation/nori.gradle

@@ -81,5 +81,7 @@ configure(project(":lucene:analysis:nori")) {
         })
       }
     }
+
+    regenerate.dependsOn compileMecabKo
   }
 }

gradle/generation/regenerate.gradle

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Create common 'regenerate' task sub-tasks can hook into.
+
+configure([
+        project(":lucene:analysis:common"),
+        project(":lucene:core"),
+        project(":lucene:analysis:icu"),
+        project(":lucene:queryparser"),
+        project(":lucene:analysis:kuromoji"),
+        project(":lucene:analysis:nori")
+]) {
+    task regenerate() {
+      description "Rerun any code or static data generation tasks."
+      group "generation"
+    }
+
+    // Make sure 'tidy' and its dependencies run after any other task in the dependencies
+    // of 'regenerate'. This ensures proper execution ordering so that tidy tasks run
+    // after whatever has been generated is complete.
+    afterEvaluate {
+      Set<Task> deps = regenerate.getTaskDependencies().getDependencies(regenerate)
+      def tidy = deps.find { it.name == "tidy" }
+      if (tidy) {
+        TaskDependency dep = tidy.taskDependencies
+        Set<Task> visited = new HashSet<>()
+        Queue<Task> queue = new ArrayDeque<>()
+        queue.add(tidy)
+        while (!queue.isEmpty()) {
+          Task t = queue.removeFirst()
+          if (visited.add(t)) {
+            queue.addAll(dep.getDependencies(t))
+          }
+        }
+        
+        def intersection = visited.intersect(deps)
+        def tidyDeps = visited - intersection + [tidy]
+        def genDeps = deps - intersection
+
+        tidyDeps.each { Task t -> t.mustRunAfter(genDeps) }
+      }
+    }
+}

gradle/generation/snowball.gradle

@@ -19,15 +19,6 @@ import org.apache.tools.ant.taskdefs.condition.Os
 
 apply plugin: "de.undercouch.download"
 
-configure(rootProject) {
-  task snowball()  {
-    description "Regenerate snowball-based sources, stopwords, and tests for ...lucene/analysis."
-    group "generation"
-
-    dependsOn ":lucene:analysis:common:snowballGen"
-  }
-}
-
 configure(project(":lucene:analysis:common")) {
   ext {
     // git commit hash of source code https://github.com/snowballstem/snowball/
@@ -47,20 +38,28 @@ configure(project(":lucene:analysis:common")) {
     snowballScript     = rootProject.file("gradle/generation/snowball.sh")
   }
 
+  def unpackFromZip = { zipFile, targetDir ->
+    project.sync {
+      from(zipTree(zipFile), {
+        eachFile { fcd ->
+          fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1))
+        }
+      })
+      into targetDir
+    }
+  }
+
   // downloads snowball stemmers (or use cached copy)
   task downloadSnowballStemmers(type: Download) {
     inputs.file(snowballPatchFile)
     src "https://github.com/snowballstem/snowball/archive/${snowballStemmerCommit}.zip"
-    def snowballStemmerZip = file("${snowballStemmerDir}.zip")
-    dest snowballStemmerZip
+    dest file("${snowballStemmerDir}.zip")
     overwrite false
     tempAndMove true
 
     doLast {
-      ant.unzip(src: snowballStemmerZip, dest: snowballStemmerDir, overwrite: "true") {
-        ant.cutdirsmapper(dirs: "1")
-      }
-      ant.patch(patchfile: snowballPatchFile, dir: snowballStemmerDir, strip: "1")
+      unpackFromZip(dest, snowballStemmerDir)
+      ant.patch(patchfile: snowballPatchFile, dir: snowballStemmerDir, strip: "1", failonerror: true)
     }
   }
 
@@ -73,9 +72,7 @@ configure(project(":lucene:analysis:common")) {
     tempAndMove true
 
     doLast {
-      ant.unzip(src: snowballWebsiteZip, dest: snowballWebsiteDir, overwrite: "true") {
-        ant.cutdirsmapper(dirs: "1")
-      }
+      unpackFromZip(snowballWebsiteZip, snowballWebsiteDir)
     }
   }
 
@@ -88,27 +85,34 @@ configure(project(":lucene:analysis:common")) {
     tempAndMove true
 
     doLast {
-      ant.unzip(src: snowballDataZip, dest: snowballDataDir, overwrite: "true") {
-        ant.cutdirsmapper(dirs: "1")
-      }
+      unpackFromZip(snowballDataZip, snowballDataDir)
     }
   }
 
   // runs shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords.
-  task snowballGen() {
-    dependsOn downloadSnowballStemmers
-    dependsOn downloadSnowballWebsite
-    dependsOn downloadSnowballData
+  task snowball() {
+    description "Regenerates snowball stemmers."
+    group "generation"
 
-    doLast {
-      if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-        throw GradleException("Snowball generation does not work on Windows, use a platform where bash is available.")
+    // Don't even bother adding dependencies on Windows.
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+      doFirst {
+        // Just emit a big fat error message but don't fail the build.
+        logger.error("Snowball generation does not work on Windows (patch and bash must be available).")
       }
+    } else {
+      dependsOn downloadSnowballStemmers
+      dependsOn downloadSnowballWebsite
+      dependsOn downloadSnowballData
 
-      project.exec {
-        executable "bash"
-        args = [snowballScript, snowballStemmerDir, snowballWebsiteDir, snowballDataDir, projectDir]
+      doFirst {
+        project.exec {
+          executable "bash"
+          args = [snowballScript, snowballStemmerDir, snowballWebsiteDir, snowballDataDir, projectDir]
+        }
       }
     }
   }
+
+  regenerate.dependsOn snowball, "tidy"
 }

gradle/generation/util.gradle

@@ -14,22 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 apply plugin: "de.undercouch.download"
 
-configure(rootProject) {
+configure(project(":lucene:core")) {
   ext {
     momanDir = file("${buildDir}/moman")
   }
 
-  task moman()  {
-    description "Regenerate Moman-based sources for ...lucene/util/automaton and ...lucene/util/packed."
-    group "generation"
-
-    dependsOn ":lucene:core:utilGenPacked"
-    dependsOn ":lucene:core:utilGenLev"
-  }
-
   task installMoman(type: Download) {
     def momanZip = file("${momanDir}/moman.zip")
 
@@ -43,10 +34,8 @@ configure(rootProject) {
       }
     }
   }
-}
 
-configure(project(":lucene:core")) {
-  task utilGenPacked(dependsOn: rootProject.installMoman) {
+  task utilGenPacked(dependsOn: installMoman) {
     description "Regenerate util/PackedBulkOperationsPacked*.java and Packed64SingleBlock.java"
     group "generation"
 
@@ -71,7 +60,7 @@ configure(project(":lucene:core")) {
     }
   }
 
-  task utilGenLev(dependsOn: rootProject.installMoman) {
+  task utilGenLev(dependsOn: installMoman) {
     description "Regenerate util/automaton Lev*ParametricDescription.java"
     group "generation"
 
@@ -96,20 +85,12 @@ configure(project(":lucene:core")) {
     }
   }
 
-  task regenerate() {
-    description "Regenerate any generated sources"
+  task moman()  {
+    description "Regenerate Moman-based sources."
     group "generation"
 
-    // Run regeneration tasks.
-    dependsOn utilGenPacked
-    dependsOn utilGenLev
-
-    // Clean up and reformat the generated sources after generation.
-    dependsOn "tidy"
+    dependsOn utilGenLev, utilGenPacked
   }
 
-  // Make sure tidy runs after generation, if they're defined.
-  tasks.matching { it.name == "tidy" }.configureEach {
-    mustRunAfter utilGenPacked, utilGenLev
-  }
+  regenerate.dependsOn moman, "tidy"
 }