LUCENE-9544: Port Nori dictionary compilation (#1926)

build.gradle

@@ -150,6 +150,7 @@ apply from: file('gradle/generation/javacc.gradle')
 apply from: file('gradle/generation/util.gradle')
 apply from: file('gradle/generation/snowball.gradle')
 apply from: file('gradle/generation/kuromoji.gradle')
+apply from: file('gradle/generation/nori.gradle')
 
 // Additional development aids.
 apply from: file('gradle/maven/maven-local.gradle')

gradle/generation/nori.gradle

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This downloads and compiles Nori dictionaries.
+
+def recompileDictionary(project, dictionaryName, Closure closure) {
+  project.javaexec {
+    main = "org.apache.lucene.analysis.ko.util.DictionaryBuilder"
+    classpath = project.sourceSets.main.runtimeClasspath
+
+    jvmArgs '-Xmx1G'
+
+    with closure
+  }
+  project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")
+}
+
+configure(project(":lucene:analysis:nori")) {
+  apply plugin: 'java-library'
+  apply plugin: "de.undercouch.download"
+
+  ext {
+    targetDir = file("src/resources")
+  }
+
+  task deleteDictionaryData() {
+    // There should really be just one but since we don't know which
+    // one it'll be, let's process all of them.
+    doFirst {
+      sourceSets.main.resources.srcDirs.each { location ->
+        delete fileTree(dir: location, include: "org/apache/lucene/analysis/ko/dict/*.dat")
+      }
+    }
+  }
+
+  task compileMecabKo(type: Download) {
+    description "Recompile dictionaries from Mecab-Ko data."
+    group "generation"
+
+    dependsOn deleteDictionaryData
+    dependsOn sourceSets.main.runtimeClasspath
+
+    def dictionaryName = "mecab-ko-dic-2.0.3-20170922"
+    def dictionarySource = "https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/${dictionaryName}.tar.gz"
+    def dictionaryFile = file("${buildDir}/generate/${dictionaryName}.tar.gz")
+    def unpackedDir = file("${buildDir}/generate/${dictionaryName}")
+
+    src dictionarySource
+    dest dictionaryFile
+    onlyIfModified true
+
+    doLast {
+      // Unpack the downloaded archive.
+      delete unpackedDir
+      ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {
+        ant.cutdirsmapper(dirs: "1")
+      }
+
+      // Compile the dictionary
+      recompileDictionary(project, dictionaryName, {
+        args += [
+            unpackedDir,
+            targetDir,
+            "utf-8",
+            false
+        ]
+      })
+    }
+  }
+}

lucene/CHANGES.txt

@@ -184,6 +184,8 @@ Other
 
 * LUCENE-9497: Integrate Error Prone, a static analysis tool during compilation (Dawid Weiss, Varun Thacker)
 
+* LUCENE-9544: add regenerate gradle script for nori dictionary (Namgyu Kim)
+
 ======================= Lucene 8.7.0 =======================
 
 API Changes