lucene/gradle/documentation/render-javadoc.gradle
Robert Muir 784ede4eda
LUCENE-9215: replace checkJavaDocs.py with doclet (#1802)
This has the same logic as the previous python, but no longer relies
upon parsing HTML output, instead using java's doclet processor.

The errors are reported like "normal" javadoc errors with source file
name and line number and happen when running "gradlew javadoc"

Although the "rules" are the same as the previous python, the python had
some bugs where the checker didn't quite do exactly what we wanted, so
some fixes were applied throughout.

Co-authored-by: Dawid Weiss <dawid.weiss@carrotsearch.com>
Co-authored-by: Uwe Schindler <uschindler@apache.org>
2020-09-02 08:29:17 -04:00

598 lines
20 KiB
Groovy

import javax.annotation.Nullable
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// generate javadocs by calling javadoc tool
// see https://docs.oracle.com/en/java/javase/11/tools/javadoc.html
def resources = scriptResources(buildscript)
allprojects {
plugins.withType(JavaPlugin) {
configurations {
missingdoclet
}
dependencies {
missingdoclet "org.apache.lucene.tools:missing-doclet"
}
ext {
relativeDocPath = project.path.replaceFirst(/:\w+:/, "").replace(':', '/')
}
// We disable the default javadoc task and have our own
// javadoc rendering task below. The default javadoc task
// will just invoke 'renderJavadoc' (to allow people to call
// conventional task name).
tasks.matching { it.name == "javadoc" }.all {
enabled = false
dependsOn "renderJavadoc"
}
task renderJavadoc(type: RenderJavadocTask) {
description "Generates Javadoc API documentation for the main source code. This directly invokes javadoc tool."
group "documentation"
taskResources = resources
dependsOn sourceSets.main.compileClasspath
classpath = sourceSets.main.compileClasspath
srcDirSet = sourceSets.main.java
outputDir = project.javadoc.destinationDir
}
task renderSiteJavadoc(type: RenderJavadocTask) {
description "Generates Javadoc API documentation for the site (relative links)."
group "documentation"
taskResources = resources
dependsOn sourceSets.main.compileClasspath
classpath = sourceSets.main.compileClasspath;
srcDirSet = sourceSets.main.java;
relativeProjectLinks = true
// Place the documentation under Lucene or Solr's documentation directory.
// docroot is defined in 'documentation.gradle'
outputDir = project.docroot.toPath().resolve(project.relativeDocPath).toFile()
}
}
}
// Set up titles and link up some offline docs for all documentation
// (they may be unused but this doesn't do any harm).
def javaJavadocPackages = rootProject.file("${resources}/java11/")
def junitJavadocPackages = rootProject.file("${resources}/junit/")
allprojects {
project.tasks.withType(RenderJavadocTask) {
title = "${project.path.startsWith(':lucene') ? 'Lucene' : 'Solr'} ${project.version} ${project.name} API"
offlineLinks += [
"https://docs.oracle.com/en/java/javase/11/docs/api/": javaJavadocPackages,
"https://junit.org/junit4/javadoc/4.12/": junitJavadocPackages
]
// Set up custom doclet.
dependsOn configurations.missingdoclet
docletpath = configurations.missingdoclet
}
}
// Configure project-specific tweaks and to-dos.
configure(project(":lucene:analysis:common")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "class"
// TODO: clean up split packages
javadocMissingIgnore = [ "org.apache.lucene.analysis.standard" ]
}
}
configure([
project(":lucene:analysis:kuromoji"),
project(":lucene:analysis:nori"),
project(":lucene:analysis:opennlp"),
project(":lucene:analysis:smartcn"),
project(":lucene:benchmark"),
project(":lucene:codecs"),
project(":lucene:grouping"),
project(":lucene:highlighter"),
project(":lucene:luke"),
project(":lucene:monitor"),
project(":lucene:queries"),
project(":lucene:queryparser"),
project(":lucene:replicator"),
project(":lucene:spatial-extras"),
]) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "class"
}
}
configure([
project(":lucene:analysis:icu"),
project(":lucene:analysis:morfologik"),
project(":lucene:analysis:phonetic"),
project(":lucene:analysis:stempel"),
project(":lucene:classification"),
project(":lucene:demo"),
project(":lucene:expressions"),
project(":lucene:facet"),
project(":lucene:join"),
project(":lucene:spatial3d"),
project(":lucene:suggest"),
]) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing @param tags
javadocMissingLevel = "method"
}
}
configure(project(":lucene:analysis:icu")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.lucene.collation",
"org.apache.lucene.collation.tokenattributes"
]
}
}
configure(project(":lucene:backward-codecs")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing @param tags
javadocMissingLevel = "method"
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.lucene.codecs",
"org.apache.lucene.codecs.lucene50",
"org.apache.lucene.codecs.lucene60",
"org.apache.lucene.codecs.lucene80",
"org.apache.lucene.codecs.lucene84",
"org.apache.lucene.codecs.lucene86"
]
}
}
configure(project(":lucene:test-framework")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "class"
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.lucene.analysis",
"org.apache.lucene.analysis.standard",
"org.apache.lucene.codecs",
"org.apache.lucene.codecs.blockterms",
"org.apache.lucene.codecs.bloom",
"org.apache.lucene.codecs.compressing",
"org.apache.lucene.codecs.uniformsplit",
"org.apache.lucene.codecs.uniformsplit.sharedterms",
"org.apache.lucene.geo",
"org.apache.lucene.index",
"org.apache.lucene.search",
"org.apache.lucene.search.similarities",
"org.apache.lucene.search.spans",
"org.apache.lucene.store",
"org.apache.lucene.util",
"org.apache.lucene.util.automaton",
"org.apache.lucene.util.fst"
]
}
}
configure(project(":lucene:sandbox")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "class"
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.lucene.search",
"org.apache.lucene.document"
]
}
}
configure(project(":lucene:misc")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "class"
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.lucene.search",
"org.apache.lucene.search.similarity",
"org.apache.lucene.util",
"org.apache.lucene.util.fst",
"org.apache.lucene.store",
"org.apache.lucene.document",
"org.apache.lucene.index"
]
}
}
configure(project(":lucene:core")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "class"
// some packages are fixed already
javadocMissingMethod = [
"org.apache.lucene.util.automaton",
"org.apache.lucene.analysis",
"org.apache.lucene.analysis.standard",
"org.apache.lucene.analysis.tokenattributes",
"org.apache.lucene.document",
"org.apache.lucene.search.similarities",
"org.apache.lucene.index",
"org.apache.lucene.codecs",
"org.apache.lucene.codecs.lucene50",
"org.apache.lucene.codecs.lucene60",
"org.apache.lucene.codecs.lucene80",
"org.apache.lucene.codecs.lucene84",
"org.apache.lucene.codecs.lucene86",
"org.apache.lucene.codecs.lucene87",
"org.apache.lucene.codecs.perfield"
]
}
}
configure(project(":solr").allprojects) {
project.tasks.withType(RenderJavadocTask) {
// TODO: fix missing javadocs
javadocMissingLevel = "package"
}
}
configure(project(":solr:contrib:velocity")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [ "org.apache.solr.response" ]
}
}
configure(project(":solr:contrib:analysis-extras")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.solr.schema",
"org.apache.solr.update.processor"
]
}
}
configure(project(":solr:contrib:analytics")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.solr.handler",
"org.apache.solr.handler.component",
"org.apache.solr.response"
]
}
}
configure(project(":solr:contrib:langid")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [ "org.apache.solr.update.processor" ]
}
}
configure(project(":solr:solrj")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [ "org.apache.solr.client.solrj.embedded" ]
}
}
configure(project(":solr:test-framework")) {
project.tasks.withType(RenderJavadocTask) {
// TODO: clean up split packages
javadocMissingIgnore = [
"org.apache.solr",
"org.apache.solr.analysis",
"org.apache.solr.cloud",
"org.apache.solr.core",
"org.apache.solr.handler.component",
"org.apache.solr.update.processor",
"org.apache.solr.util"
]
}
}
// Fix for Java 11 Javadoc tool that cannot handle split packages between modules correctly.
// (by removing all the packages which are part of lucene-core)
// See: https://issues.apache.org/jira/browse/LUCENE-8738?focusedCommentId=16818106&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16818106
configure(subprojects.findAll { it.path.startsWith(':lucene') && it.path != ':lucene:core' }) {
project.tasks.withType(RenderJavadocTask) {
doLast {
Set luceneCorePackages = file("${project(':lucene:core').tasks[name].outputDir}/element-list").readLines('UTF-8').toSet();
File elementFile = file("${outputDir}/element-list");
List elements = elementFile.readLines('UTF-8');
elements.removeAll(luceneCorePackages)
elementFile.write(elements.join('\n').concat('\n'), 'UTF-8');
}
}
}
configure(project(':lucene:demo')) {
project.tasks.withType(RenderJavadocTask) {
// For the demo, we link the example source in the javadocs, as it's ref'ed elsewhere
linksource = true
}
}
// Disable Javadoc rendering for these projects.
configure(subprojects.findAll { it.path in [
':solr:solr-ref-guide',
':solr:server',
':solr:webapp']}) {
project.tasks.withType(RenderJavadocTask) {
enabled = false
}
}
// Add cross-project documentation task dependencies:
// - each RenderJavaDocs task gets a dependency to all tasks with same name in its dependencies
// - the dependency is using dependsOn with a closure to enable lazy evaluation
configure(subprojects) {
project.tasks.withType(RenderJavadocTask) { task ->
task.dependsOn {
task.project.configurations.implementation.allDependencies.withType(ProjectDependency).collect { dep ->
def otherProject = dep.dependencyProject
return otherProject.tasks.findByName(task.name)
}
}
}
}
class RenderJavadocTask extends DefaultTask {
@InputFiles
@SkipWhenEmpty
SourceDirectorySet srcDirSet;
@OutputDirectory
File outputDir
@CompileClasspath
FileCollection classpath
@CompileClasspath
FileCollection docletpath
@Input
String title
@Input
boolean linksource = false
@Input
boolean relativeProjectLinks = false
@Input
def offlineLinks = [:]
@Input
def luceneDocUrl = "${->project.luceneDocUrl}"
@Input
def solrDocUrl = "${->project.solrDocUrl}"
// default is to require full javadocs
@Input
String javadocMissingLevel = "parameter"
// anything in these packages is checked with level=method. This allows iteratively fixing one package at a time.
@Input
List<String> javadocMissingMethod = []
// default is not to ignore any elements, should only be used to workaround split packages
@Input
List<String> javadocMissingIgnore = []
@Nullable
@Optional
@Input
def executable
@Input
def taskResources
/** Utility method to recursively collect all tasks with same name like this one that we depend on */
private Set findRenderTasksInDependencies() {
Set found = []
def collectDeps
collectDeps = { task -> task.taskDependencies.getDependencies(task).findAll{ it.name == this.name && it.enabled && !found.contains(it) }.each{
found << it
collectDeps(it)
}}
collectDeps(this)
return found
}
@TaskAction
public void render() {
def srcDirs = srcDirSet.srcDirs.findAll { dir -> dir.exists() }
def optionsFile = project.file("${getTemporaryDir()}/javadoc-options.txt")
// create the directory, so relative link calculation knows that it's a directory:
outputDir.mkdirs();
def opts = []
opts << [ '-overview', project.file("${srcDirs[0]}/overview.html") ]
opts << [ '-sourcepath', srcDirs.join(File.pathSeparator) ]
opts << [ '-subpackages', project.path.startsWith(':lucene') ? 'org.apache.lucene' : 'org.apache.solr' ]
opts << [ '-d', outputDir ]
opts << '-protected'
opts << [ '-encoding', 'UTF-8' ]
opts << [ '-charset', 'UTF-8' ]
opts << [ '-docencoding', 'UTF-8' ]
opts << '-noindex'
opts << '-author'
opts << '-version'
if (linksource) {
opts << '-linksource'
}
opts << '-use'
opts << [ '-locale', 'en_US' ]
opts << [ '-windowtitle', title ]
opts << [ '-doctitle', title ]
if (!classpath.isEmpty()) {
opts << [ '-classpath', classpath.asPath ]
}
opts << [ '-bottom', "<i>Copyright &copy; 2000-${project.buildYear} Apache Software Foundation. All Rights Reserved.</i>" ]
opts << [ '-tag', 'lucene.experimental:a:WARNING: This API is experimental and might change in incompatible ways in the next release.' ]
opts << [ '-tag', 'lucene.internal:a:NOTE: This API is for internal purposes only and might change in incompatible ways in the next release.' ]
opts << [ '-tag', "lucene.spi:t:SPI Name (case-insensitive: if the name is 'htmlStrip', 'htmlstrip' can be used when looking up the service)." ]
opts << [ '-doclet', "org.apache.lucene.missingdoclet.MissingDoclet" ]
opts << [ '-docletpath', docletpath.asPath ]
opts << [ '--missing-level', javadocMissingLevel ]
if (javadocMissingIgnore) {
opts << [ '--missing-ignore', String.join(',', javadocMissingIgnore) ]
}
if (javadocMissingMethod) {
opts << [ '--missing-method', String.join(',', javadocMissingMethod) ]
}
opts << [ '-quiet' ]
def allOfflineLinks = [:]
allOfflineLinks.putAll(offlineLinks)
// Resolve inter-project links:
// - find all (enabled) tasks this tasks depends on (with same name), calling findRenderTasksInDependencies()
// - sort the tasks preferring those whose project name equals 'core', then lexigraphical by path
// - for each task get output dir to create relative or absolute link
// NOTE: explicitly exclude solr/test-framework, or attempting to link to lucene-test-framework because if we did javadoc would
// attempt to link class refs in in org.apache.lucene, causing broken links. (either broken links to things like "Directory" if
// lucene-test-framework was first, or broken links to things like LuceneTestCase if lucene-core was first)
if (project.path != ':solr:test-framework') { //
findRenderTasksInDependencies()
.sort(false, Comparator.comparing { (it.project.name != 'core') as Boolean }.thenComparing(Comparator.comparing { it.path }))
.each { otherTask ->
def otherProject = otherTask.project
// For relative links we compute the actual relative link between projects.
def crossLuceneSolr = (otherProject.docroot != project.docroot)
if (relativeProjectLinks && !crossLuceneSolr) {
def pathTo = otherTask.outputDir.toPath().toAbsolutePath()
def pathFrom = outputDir.toPath().toAbsolutePath()
def relative = pathFrom.relativize(pathTo).toString().replace(File.separator, '/')
opts << ['-link', relative]
} else {
// For absolute links, we determine the target URL by assembling the full URL.
def base = otherProject.path.startsWith(":lucene") ? luceneDocUrl : solrDocUrl
allOfflineLinks.put("${base}/${otherProject.relativeDocPath}/".toString(), otherTask.outputDir)
}
}
}
// Add offline links.
allOfflineLinks.each { url, dir ->
// Some sanity check/ validation here to ensure dir/package-list or dir/element-list is present.
if (!project.file("$dir/package-list").exists() &&
!project.file("$dir/element-list").exists()) {
throw new GradleException("Expected pre-rendered package-list or element-list at ${dir}.")
}
opts << [ '-linkoffline', url, dir ]
}
opts << [ '--release', 11 ]
opts << '-Xdoclint:all,-missing'
// Temporary file that holds all javadoc options for the current task.
optionsFile.withWriter("UTF-8", { writer ->
// escapes an option with single quotes or whitespace to be passed in the options.txt file for
def escapeJavadocOption = { String s -> (s =~ /[ '"]/) ? ("'" + s.replaceAll(/[\\'"]/, /\\$0/) + "'") : s }
opts.each { entry ->
if (entry instanceof List) {
writer.write(entry.collect { escapeJavadocOption(it as String) }.join(" "))
} else {
writer.write(escapeJavadocOption(entry as String))
}
writer.write('\n')
}
})
def javadocCmd = {
if (executable == null) {
JavaInstallationRegistry registry = project.extensions.getByType(JavaInstallationRegistry)
JavaInstallation currentJvm = registry.installationForCurrentVirtualMachine.get()
return currentJvm.jdk.get().javadocExecutable.asFile
} else {
return project.file(executable)
}
}()
logger.info("Javadoc executable used: ${javadocCmd}")
def outputFile = project.file("${getTemporaryDir()}/javadoc-output.txt")
def result
outputFile.withOutputStream { output ->
result = project.exec {
executable javadocCmd
standardOutput = output
errorOutput = output
args += [ "@${optionsFile}" ]
// -J flags can't be passed via options file... (an error "javadoc: error - invalid flag: -J-Xmx512m" occurs.)
args += [ "-J-Xmx512m" ]
// force locale to be "en_US" (fix for: https://bugs.openjdk.java.net/browse/JDK-8222793)
args += [ "-J-Duser.language=en", "-J-Duser.country=US" ]
ignoreExitValue true
}
}
if (result.getExitValue() != 0) {
// Pipe the output to console. Intentionally skips any encoding conversion
// and pumps raw bytes.
System.out.write(outputFile.bytes)
def cause
try {
result.rethrowFailure()
} catch (ex) {
cause = ex
}
throw new GradleException("Javadoc generation failed for ${project.path},\n Options file at: ${optionsFile}\n Command output at: ${outputFile}", cause)
}
// append some special table css, prettify css
ant.concat(destfile: "${outputDir}/stylesheet.css", append: "true", fixlastline: "true", encoding: "UTF-8") {
filelist(dir: taskResources, files: "table_padding.css")
filelist(dir: project.file("${taskResources}/prettify"), files: "prettify.css")
}
// append prettify to scripts
ant.concat(destfile: "${outputDir}/script.js", append: "true", fixlastline: "true", encoding: "UTF-8") {
filelist(dir: project.file("${taskResources}/prettify"), files: "prettify.js inject-javadocs.js")
}
ant.fixcrlf(srcdir: outputDir, includes: "stylesheet.css script.js", eol: "lf", fixlast: "true", encoding: "UTF-8")
}
}