LUCENE-9900: Regenerate/ run ICU only if inputs changed (#61)

This commit is contained in:
Dawid Weiss 2021-04-02 11:46:43 +02:00 committed by GitHub
parent e3ae57a3c1
commit 010e3a1ba9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 67 additions and 25 deletions

View File

@ -43,12 +43,21 @@ configure(project(":lucene:analysis:icu")) {
icupkg = file("${icuBinDir}/icupkg") icupkg = file("${icuBinDir}/icupkg")
} }
def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
task genUtr30DataFiles() { task genUtr30DataFiles() {
dependsOn Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux" dependsOn icuCompileTask
// May be undefined yet, so use a provider. // May be undefined yet, so use a provider.
dependsOn { sourceSets.tools.runtimeClasspath } dependsOn { sourceSets.tools.runtimeClasspath }
// gennorm generates file order-dependent output, so make it constant here.
def inputFiles = fileTree(dir: utr30DataDir, include: "*.txt").asList().toSorted(Comparator.comparing { File f -> f.name })
def outputFile = file("src/resources/org/apache/lucene/analysis/icu/utr30.nrm")
inputs.files inputFiles
outputs.file outputFile
doFirst { doFirst {
// all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm
def v = getVersion('com.ibm.icu', 'icu4j'); def v = getVersion('com.ibm.icu', 'icu4j');
@ -71,9 +80,7 @@ configure(project(":lucene:analysis:icu")) {
utr30DataDir, utr30DataDir,
"-o", "-o",
"${buildDir}/utr30.tmp", "${buildDir}/utr30.tmp",
"nfc.txt", "nfkc.txt", "nfkc_cf.txt", "BasicFoldings.txt", *(inputFiles.collect { it.name })
"DiacriticFolding.txt", "DingbatFolding.txt", "HanRadicalFolding.txt",
"NativeDigitFolding.txt"
] ]
} }
@ -82,7 +89,7 @@ configure(project(":lucene:analysis:icu")) {
args = [ args = [
"-tb", "-tb",
"${buildDir}/utr30.tmp", "${buildDir}/utr30.tmp",
"src/resources/org/apache/lucene/analysis/icu/utr30.nrm" outputFile
] ]
} }
} }
@ -92,6 +99,12 @@ configure(project(":lucene:analysis:icu")) {
// May be undefined yet, so use a provider. // May be undefined yet, so use a provider.
dependsOn { sourceSets.tools.runtimeClasspath } dependsOn { sourceSets.tools.runtimeClasspath }
def sourceDir = file("src/data/uax29")
def targetDir = file("src/resources/org/apache/lucene/analysis/icu/segmentation")
inputs.files fileTree(dir: sourceDir, include: "*.rbbi")
outputs.files fileTree(dir: targetDir, include: "*.brk")
doFirst { doFirst {
project.javaexec { project.javaexec {
main = "org.apache.lucene.analysis.icu.RBBIRuleCompiler" main = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"
@ -99,15 +112,13 @@ configure(project(":lucene:analysis:icu")) {
ignoreExitValue false ignoreExitValue false
enableAssertions true enableAssertions true
args = [ args = [ sourceDir, targetDir ]
"src/data/uax29",
"src/resources/org/apache/lucene/analysis/icu/segmentation"
]
} }
} }
} }
regenerate.dependsOn genUtr30DataFiles, genRbbi regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFiles, [ ignoreWithSource: icuCompileTask ])
regenerate.dependsOn wrapWithPersistentChecksums(genRbbi)
task compileIcuWindows() { task compileIcuWindows() {
doFirst { doFirst {

View File

@ -252,9 +252,9 @@ configure(project(":lucene:queryparser")) {
description "Regenerate query parsers (javacc syntax definitions)." description "Regenerate query parsers (javacc syntax definitions)."
group "generation" group "generation"
dependsOn wrapWithPersistentChecksums(javaccParserClassic, "spotlessApply"), dependsOn wrapWithPersistentChecksums(javaccParserClassic, [ andThenTasks: "spotlessApply" ]),
wrapWithPersistentChecksums(javaccParserSurround, "spotlessApply"), wrapWithPersistentChecksums(javaccParserSurround, [ andThenTasks: "spotlessApply" ]),
wrapWithPersistentChecksums(javaccParserFlexible, "spotlessApply") wrapWithPersistentChecksums(javaccParserFlexible, [ andThenTasks: "spotlessApply" ])
} }
regenerate.dependsOn javacc regenerate.dependsOn javacc

View File

@ -49,7 +49,7 @@ configure(project(":lucene:core")) {
} }
} }
regenerate.dependsOn wrapWithPersistentChecksums(generateStandardTokenizer, "spotlessApply") regenerate.dependsOn wrapWithPersistentChecksums(generateStandardTokenizer, [ andThenTasks: "spotlessApply" ])
} }
configure(project(":lucene:analysis:common")) { configure(project(":lucene:analysis:common")) {
@ -118,10 +118,10 @@ configure(project(":lucene:analysis:common")) {
} }
} }
regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizer, "spotlessApply"), regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizer, [ andThenTasks: "spotlessApply" ]),
wrapWithPersistentChecksums(generateClassicTokenizer, "spotlessApply"), wrapWithPersistentChecksums(generateClassicTokenizer, [ andThenTasks: "spotlessApply" ]),
wrapWithPersistentChecksums(generateUAX29URLEmailTokenizer, "spotlessApply"), wrapWithPersistentChecksums(generateUAX29URLEmailTokenizer, [ andThenTasks: "spotlessApply" ]),
wrapWithPersistentChecksums(generateHTMLStripCharFilter, "spotlessApply") wrapWithPersistentChecksums(generateHTMLStripCharFilter, [ andThenTasks: "spotlessApply" ])
} }
class JFlexTask extends DefaultTask { class JFlexTask extends DefaultTask {

View File

@ -99,8 +99,8 @@ configure(project(":lucene:core")) {
description "Regenerate Moman-based sources." description "Regenerate Moman-based sources."
group "generation" group "generation"
dependsOn wrapWithPersistentChecksums(utilGenPacked, "spotlessApply") dependsOn wrapWithPersistentChecksums(utilGenPacked, [ andThenTasks: "spotlessApply" ])
dependsOn wrapWithPersistentChecksums(utilGenLev, "spotlessApply") dependsOn wrapWithPersistentChecksums(utilGenLev, [ andThenTasks: "spotlessApply" ])
} }
regenerate.dependsOn moman regenerate.dependsOn moman

View File

@ -56,9 +56,23 @@ configure([
// constraints to each node in the execution graph but not node-and-dependencies). // constraints to each node in the execution graph but not node-and-dependencies).
// //
// sourceTask - the task to wrap // sourceTask - the task to wrap
// otherTasks - other tasks that should be scheduled to run after source task and // extraConfig - a map with extra (optional) configuration options.
// before checksum calculation. // andThenTasks: other tasks that should be scheduled to run after source task and
wrapWithPersistentChecksums = { Task sourceTask, Object... otherTasks -> // before checksum calculation.
wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
def toList = { value ->
if (value instanceof List) {
return value
} else if (value == null) {
return []
} else {
return [ value ]
}
}
List<Object> andThenTasks = toList(extraConfig.get("andThenTasks"))
List<Object> ignoreWithSource = toList(extraConfig.get("ignoreWithSource"))
// Create checksum-loader task. // Create checksum-loader task.
Task checksumLoadTask = tasks.create("${sourceTask.name}ChecksumLoad", { Task checksumLoadTask = tasks.create("${sourceTask.name}ChecksumLoad", {
ext { ext {
@ -140,7 +154,7 @@ configure([
def deps = [ def deps = [
checksumLoadTask, checksumLoadTask,
sourceTask, sourceTask,
*otherTasks, *andThenTasks,
checksumSaveTask checksumSaveTask
].flatten() ].flatten()
@ -163,7 +177,7 @@ configure([
// Set conditional execution only if checksum mismatch occurred. // Set conditional execution only if checksum mismatch occurred.
if (!gradle.startParameter.isRerunTasks()) { if (!gradle.startParameter.isRerunTasks()) {
project.afterEvaluate { project.afterEvaluate {
resolveTaskRefs([sourceTask, checksumSaveTask]).each { t -> resolveTaskRefs([sourceTask, *ignoreWithSource, checksumSaveTask]).each { t ->
t.configure { t.configure {
logger.info("Making " + t.name + " run only if " + checksumLoadTask.name + " indicates changes") logger.info("Making " + t.name + " run only if " + checksumLoadTask.name + " indicates changes")
onlyIf { !checksumLoadTask.checksumMatch } onlyIf { !checksumLoadTask.checksumMatch }

View File

@ -0,0 +1,6 @@
{
"lucene/analysis/icu/src/data/uax29/Default.rbbi": "71bfaee5e81ac272aff828d1e44d0612be1b8363",
"lucene/analysis/icu/src/data/uax29/MyanmarSyllable.rbbi": "4c6817658b454add5ec1f9ac8c0015ce8eb3b5f2",
"lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk": "1b9013b7ef4ba32a851a330c58a8fa820b9dda79",
"lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk": "cc023ec17e0148518086098691785a32b88ee09a"
}

View File

@ -0,0 +1,11 @@
{
"lucene/analysis/icu/src/data/utr30/BasicFoldings.txt": "fc5badae307c5b740f8dd81407e4be04e3e2fde6",
"lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt": "2d48c24758ed9322b49eee84c23c74b7f11337c6",
"lucene/analysis/icu/src/data/utr30/DingbatFolding.txt": "ca562b2029e208bdf07cb2e5f689e509a336265e",
"lucene/analysis/icu/src/data/utr30/HanRadicalFolding.txt": "f5848aa38cbf1a11131d59384f31477f3b7f077b",
"lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt": "434ad7eb17fc32a441385795e4a4aa7e589ec30a",
"lucene/analysis/icu/src/data/utr30/nfc.txt": "ec95d7f7e5910791717234dd09efc4b13dc32d35",
"lucene/analysis/icu/src/data/utr30/nfkc.txt": "b8e91bc64e354af505d51f6072a43c90c6b4d1b5",
"lucene/analysis/icu/src/data/utr30/nfkc_cf.txt": "22d90ea4a7771e6ddebaaeb9438e98ce625e16f5",
"lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm": "32a6a3198039883c93f9ebef31fe24c1029f2b07"
}