LUCENE-9134: lucene:core:jflexStandardTokenizerImpl

build.gradle

@@ -22,7 +22,8 @@ ext {
   // Workaround for this one, for now:
   // https://github.com/palantir/gradle-consistent-versions/issues/383
   scriptDepVersions = [
-      "apache-rat": "0.11"
+      "apache-rat": "0.11",
+      "jflex": "1.7.0"
   ]
 }
 
@@ -61,6 +62,9 @@ apply from: file('gradle/validation/config-file-sanity.gradle')
 apply from: file('gradle/validation/rat-sources.gradle')
 apply from: file('gradle/validation/owasp-dependency-check.gradle')
 
+// Source or data regeneration tasks
+apply from: file('gradle/generation/jflex.gradle')
+
 // Additional development aids.
 apply from: file('gradle/maven/maven-local.gradle')
 apply from: file('gradle/testing/per-project-summary.gradle')

gradle/generation/jflex.gradle

@@ -0,0 +1,89 @@
+
+// Add a top-level pseudo-task to which we will attach individual regenerate tasks.
+
+configure(rootProject) {
+  configurations {
+    jflex
+  }
+
+  dependencies {
+    jflex "de.jflex:jflex:${scriptDepVersions['jflex']}"
+  }
+
+  task jflex() {
+    description "Regenerate sources for corresponding jflex grammar files."
+    group "generation"
+
+    dependsOn ":lucene:core:jflexStandardTokenizerImpl"
+  }
+}
+
+// We always regenerate, no need to declare outputs.
+class JFlexTask extends DefaultTask {
+  @Input
+  File jflexFile
+
+  @Input
+  File skeleton
+
+  JFlexTask() {
+    dependsOn(project.rootProject.configurations.jflex)
+  }
+
+  @TaskAction
+  def generate() {
+    if (!jflexFile || !jflexFile.exists()) {
+      throw new RuntimeException("JFlex file does not exist: ${jflexFile}")
+    }
+    def targetDir = jflexFile.parentFile
+    def target = jflexFile.absolutePath.replace(".jflex", ".java")
+
+    logger.lifecycle("Regenerating JFlex:\n  from: ${jflexFile}\n    to: ${target}")
+    project.javaexec {
+      classpath {
+        project.rootProject.configurations.jflex
+      }
+      main = "jflex.Main"
+      args += [
+          "-nobak",
+          "--quiet",
+          "--encoding", "UTF-8",
+      ]
+
+      if (skeleton) {
+        args += ["--skel", skeleton.absolutePath]
+      }
+
+      args += [
+        "-d", targetDir.absolutePath,
+        jflexFile
+      ]
+    }
+
+    // Correct line endings for Windows.
+    project.ant.fixcrlf(
+        file: target,
+        encoding: "UTF-8",
+        eol: "lf"
+    )
+  }
+}
+
+configure(project(":lucene:core")) {
+  task jflexStandardTokenizerImpl(type: JFlexTask) {
+    description "Regenerate StandardTokenizerImpl.java"
+    group "generation"
+
+    jflexFile = file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex')
+    skeleton = file("src/data/jflex/skeleton.disable.buffer.expansion.txt")
+
+    doLast {
+      ant.replace(
+          file: file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java'),
+          encoding: "UTF-8",
+          token: "private static final int ZZ_BUFFERSIZE =",
+          value: "private int ZZ_BUFFERSIZE ="
+      )
+    }
+  }
+}