diff --git a/build.gradle b/build.gradle index 8a335a510da..f56dd660f19 100644 --- a/build.gradle +++ b/build.gradle @@ -148,6 +148,7 @@ apply from: file('gradle/validation/check-broken-links.gradle') apply from: file('gradle/validation/spotless.gradle') // Source or data regeneration tasks +apply from: file('gradle/generation/regenerate.gradle') apply from: file('gradle/generation/jflex.gradle') apply from: file('gradle/generation/util.gradle') apply from: file('gradle/generation/snowball.gradle') diff --git a/gradle/generation/icu.gradle b/gradle/generation/icu.gradle index 28585d0d6dd..fca3d8a5ca8 100644 --- a/gradle/generation/icu.gradle +++ b/gradle/generation/icu.gradle @@ -108,13 +108,7 @@ configure(project(":lucene:analysis:icu")) { } } - task regenerate() { - description "Regenerate ICU data files" - group "generation" - - dependsOn genUtr30DataFiles - dependsOn genRbbi - } + regenerate.dependsOn genUtr30DataFiles, genRbbi task compileIcuWindows() { doFirst { diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle index c60c1ce2025..c0c0e70d4b5 100644 --- a/gradle/generation/javacc.gradle +++ b/gradle/generation/javacc.gradle @@ -242,22 +242,6 @@ configure(project(":lucene:queryparser")) { } } - task regenerate() { - description "Regenerate any generated sources" - group "generation" - - // Run regeneration tasks. - dependsOn javaccParserClassic, javaccParserSurround, javaccParserFlexible - - // Clean up and reformat the generated sources after generation. - dependsOn "tidy" - } - - // Make sure tidy runs after generation, if they're defined. - tasks.matching { it.name == "tidy" }.configureEach { - mustRunAfter javaccParserClassic, javaccParserSurround, javaccParserFlexible - } - task javacc() { description "Regenerate query parsers (javacc syntax definitions)." group "generation" @@ -266,6 +250,8 @@ configure(project(":lucene:queryparser")) { dependsOn javaccParserSurround dependsOn javaccParserFlexible } + + regenerate.dependsOn javacc, tidy } // We always regenerate, no need to declare outputs. diff --git a/gradle/generation/jflex.gradle b/gradle/generation/jflex.gradle index 52bf08d45c4..8f6f5ca65ea 100644 --- a/gradle/generation/jflex.gradle +++ b/gradle/generation/jflex.gradle @@ -103,6 +103,8 @@ configure(project(":lucene:core")) { ) } } + + regenerate.dependsOn jflexStandardTokenizerImpl, "tidy" } configure(project(":lucene:analysis:common")) { @@ -132,7 +134,7 @@ configure(project(":lucene:analysis:common")) { heapSize = "12g" doFirst { - logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires tons of memory).") + logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires ${heapSize} of memory!).") } doLast { @@ -175,25 +177,9 @@ configure(project(":lucene:analysis:common")) { } } - task regenerate() { - description "Regenerate any generated sources" - group "generation" - - // Run regeneration tasks. - dependsOn jflexUAX29URLEmailTokenizerImpl - dependsOn jflexHTMLStripCharFilter - dependsOn jflexClassicTokenizerImpl - dependsOn jflexWikipediaTokenizerImpl - - // Clean up and reformat the generated sources after generation. - dependsOn "tidy" - } - - // Make sure tidy runs after generation, if they're defined. - tasks.matching { it.name == "tidy" }.configureEach { - mustRunAfter jflexUAX29URLEmailTokenizerImpl, - jflexHTMLStripCharFilter, - jflexClassicTokenizerImpl, - jflexWikipediaTokenizerImpl - } + regenerate.dependsOn jflexUAX29URLEmailTokenizerImpl, + jflexHTMLStripCharFilter, + jflexClassicTokenizerImpl, + jflexWikipediaTokenizerImpl, + "tidy" } diff --git a/gradle/generation/kuromoji.gradle b/gradle/generation/kuromoji.gradle index 8dc082b233d..a18cee7eeb1 100644 --- a/gradle/generation/kuromoji.gradle +++ b/gradle/generation/kuromoji.gradle @@ -128,5 +128,8 @@ configure(project(":lucene:analysis:kuromoji")) { }) } } + + // TODO: should we include this in default regeneration? + // regenerate.dependsOn compileMecab } } diff --git a/gradle/generation/nori.gradle b/gradle/generation/nori.gradle index 78da8cc2863..4fadb11d6f7 100644 --- a/gradle/generation/nori.gradle +++ b/gradle/generation/nori.gradle @@ -81,5 +81,7 @@ configure(project(":lucene:analysis:nori")) { }) } } + + regenerate.dependsOn compileMecabKo } } diff --git a/gradle/generation/regenerate.gradle b/gradle/generation/regenerate.gradle new file mode 100644 index 00000000000..8bbd3676b56 --- /dev/null +++ b/gradle/generation/regenerate.gradle @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Create common 'regenerate' task sub-tasks can hook into. + +configure([ + project(":lucene:analysis:common"), + project(":lucene:core"), + project(":lucene:analysis:icu"), + project(":lucene:queryparser"), + project(":lucene:analysis:kuromoji"), + project(":lucene:analysis:nori") +]) { + task regenerate() { + description "Rerun any code or static data generation tasks." + group "generation" + } + + // Make sure 'tidy' and its dependencies run after any other task in the dependencies + // of 'regenerate'. This ensures proper execution ordering so that tidy tasks run + // after whatever has been generated is complete. + afterEvaluate { + Set deps = regenerate.getTaskDependencies().getDependencies(regenerate) + def tidy = deps.find { it.name == "tidy" } + if (tidy) { + TaskDependency dep = tidy.taskDependencies + Set visited = new HashSet<>() + Queue queue = new ArrayDeque<>() + queue.add(tidy) + while (!queue.isEmpty()) { + Task t = queue.removeFirst() + if (visited.add(t)) { + queue.addAll(dep.getDependencies(t)) + } + } + + def intersection = visited.intersect(deps) + def tidyDeps = visited - intersection + [tidy] + def genDeps = deps - intersection + + tidyDeps.each { Task t -> t.mustRunAfter(genDeps) } + } + } +} \ No newline at end of file diff --git a/gradle/generation/snowball.gradle b/gradle/generation/snowball.gradle index db81a563d47..5f2ff716367 100644 --- a/gradle/generation/snowball.gradle +++ b/gradle/generation/snowball.gradle @@ -19,15 +19,6 @@ import org.apache.tools.ant.taskdefs.condition.Os apply plugin: "de.undercouch.download" -configure(rootProject) { - task snowball() { - description "Regenerate snowball-based sources, stopwords, and tests for ...lucene/analysis." - group "generation" - - dependsOn ":lucene:analysis:common:snowballGen" - } -} - configure(project(":lucene:analysis:common")) { ext { // git commit hash of source code https://github.com/snowballstem/snowball/ @@ -47,20 +38,28 @@ configure(project(":lucene:analysis:common")) { snowballScript = rootProject.file("gradle/generation/snowball.sh") } + def unpackFromZip = { zipFile, targetDir -> + project.sync { + from(zipTree(zipFile), { + eachFile { fcd -> + fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1)) + } + }) + into targetDir + } + } + // downloads snowball stemmers (or use cached copy) task downloadSnowballStemmers(type: Download) { inputs.file(snowballPatchFile) src "https://github.com/snowballstem/snowball/archive/${snowballStemmerCommit}.zip" - def snowballStemmerZip = file("${snowballStemmerDir}.zip") - dest snowballStemmerZip + dest file("${snowballStemmerDir}.zip") overwrite false tempAndMove true doLast { - ant.unzip(src: snowballStemmerZip, dest: snowballStemmerDir, overwrite: "true") { - ant.cutdirsmapper(dirs: "1") - } - ant.patch(patchfile: snowballPatchFile, dir: snowballStemmerDir, strip: "1") + unpackFromZip(dest, snowballStemmerDir) + ant.patch(patchfile: snowballPatchFile, dir: snowballStemmerDir, strip: "1", failonerror: true) } } @@ -73,9 +72,7 @@ configure(project(":lucene:analysis:common")) { tempAndMove true doLast { - ant.unzip(src: snowballWebsiteZip, dest: snowballWebsiteDir, overwrite: "true") { - ant.cutdirsmapper(dirs: "1") - } + unpackFromZip(snowballWebsiteZip, snowballWebsiteDir) } } @@ -88,27 +85,34 @@ configure(project(":lucene:analysis:common")) { tempAndMove true doLast { - ant.unzip(src: snowballDataZip, dest: snowballDataDir, overwrite: "true") { - ant.cutdirsmapper(dirs: "1") - } + unpackFromZip(snowballDataZip, snowballDataDir) } } // runs shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords. - task snowballGen() { - dependsOn downloadSnowballStemmers - dependsOn downloadSnowballWebsite - dependsOn downloadSnowballData + task snowball() { + description "Regenerates snowball stemmers." + group "generation" - doLast { - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - throw GradleException("Snowball generation does not work on Windows, use a platform where bash is available.") + // Don't even bother adding dependencies on Windows. + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + doFirst { + // Just emit a big fat error message but don't fail the build. + logger.error("Snowball generation does not work on Windows (patch and bash must be available).") } + } else { + dependsOn downloadSnowballStemmers + dependsOn downloadSnowballWebsite + dependsOn downloadSnowballData - project.exec { - executable "bash" - args = [snowballScript, snowballStemmerDir, snowballWebsiteDir, snowballDataDir, projectDir] + doFirst { + project.exec { + executable "bash" + args = [snowballScript, snowballStemmerDir, snowballWebsiteDir, snowballDataDir, projectDir] + } } } } + + regenerate.dependsOn snowball, "tidy" } diff --git a/gradle/generation/util.gradle b/gradle/generation/util.gradle index 2ce785685b6..2484f44128a 100644 --- a/gradle/generation/util.gradle +++ b/gradle/generation/util.gradle @@ -14,22 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - apply plugin: "de.undercouch.download" -configure(rootProject) { +configure(project(":lucene:core")) { ext { momanDir = file("${buildDir}/moman") } - task moman() { - description "Regenerate Moman-based sources for ...lucene/util/automaton and ...lucene/util/packed." - group "generation" - - dependsOn ":lucene:core:utilGenPacked" - dependsOn ":lucene:core:utilGenLev" - } - task installMoman(type: Download) { def momanZip = file("${momanDir}/moman.zip") @@ -43,10 +34,8 @@ configure(rootProject) { } } } -} -configure(project(":lucene:core")) { - task utilGenPacked(dependsOn: rootProject.installMoman) { + task utilGenPacked(dependsOn: installMoman) { description "Regenerate util/PackedBulkOperationsPacked*.java and Packed64SingleBlock.java" group "generation" @@ -71,7 +60,7 @@ configure(project(":lucene:core")) { } } - task utilGenLev(dependsOn: rootProject.installMoman) { + task utilGenLev(dependsOn: installMoman) { description "Regenerate util/automaton Lev*ParametricDescription.java" group "generation" @@ -96,20 +85,12 @@ configure(project(":lucene:core")) { } } - task regenerate() { - description "Regenerate any generated sources" + task moman() { + description "Regenerate Moman-based sources." group "generation" - // Run regeneration tasks. - dependsOn utilGenPacked - dependsOn utilGenLev - - // Clean up and reformat the generated sources after generation. - dependsOn "tidy" + dependsOn utilGenLev, utilGenPacked } - // Make sure tidy runs after generation, if they're defined. - tasks.matching { it.name == "tidy" }.configureEach { - mustRunAfter utilGenPacked, utilGenLev - } + regenerate.dependsOn moman, "tidy" }