lucene/gradle/documentation/render-javadoc.gradle

import javax.annotation.Nullable

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// generate javadocs by calling javadoc tool
// see https://docs.oracle.com/en/java/javase/11/tools/javadoc.html

def resources = scriptResources(buildscript)

allprojects {
  plugins.withType(JavaPlugin) {
    configurations {
      missingdoclet
    }

    dependencies {
      missingdoclet "org.apache.lucene.tools:missing-doclet"
    }

    ext {
      relativeDocPath = project.path.replaceFirst(/:\w+:/, "").replace(':', '/')
    }

    // We disable the default javadoc task and have our own
    // javadoc rendering task below. The default javadoc task
    // will just invoke 'renderJavadoc' (to allow people to call
    // conventional task name).
    tasks.matching { it.name == "javadoc" }.all {
      enabled = false
      dependsOn "renderJavadoc"
    }

    task renderJavadoc(type: RenderJavadocTask) {
      description "Generates Javadoc API documentation for each module. This directly invokes javadoc tool."
      group "documentation"

      taskResources = resources
      dependsOn sourceSets.main.compileClasspath
      classpath = sourceSets.main.compileClasspath
      srcDirSet = sourceSets.main.java

      outputDir = project.javadoc.destinationDir
    }

    task renderSiteJavadoc(type: RenderJavadocTask) {
      description "Generates Javadoc API documentation for the site (relative links)."
      group "documentation"

      taskResources = resources
      dependsOn sourceSets.main.compileClasspath
      classpath = sourceSets.main.compileClasspath;
      srcDirSet = sourceSets.main.java;

      relativeProjectLinks = true

      // Place the documentation under Lucene or Solr's documentation directory.
      // docroot is defined in 'documentation.gradle'
      outputDir = project.docroot.toPath().resolve(project.relativeDocPath).toFile()
    }
  }
}

// Set up titles and link up some offline docs for all documentation
// (they may be unused but this doesn't do any harm).

def javaJavadocPackages = rootProject.file("${resources}/java11/")
def junitJavadocPackages = rootProject.file("${resources}/junit/")
allprojects {
  project.tasks.withType(RenderJavadocTask) {
    title = "Lucene ${project.version} ${project.name} API"

    offlineLinks += [
        "https://docs.oracle.com/en/java/javase/11/docs/api/": javaJavadocPackages,
        "https://junit.org/junit4/javadoc/4.12/": junitJavadocPackages
    ]

    luceneDocUrl = provider({ rootProject.luceneDocUrl })

    // Set up custom doclet.
    dependsOn configurations.missingdoclet
    docletpath = configurations.missingdoclet
  }
}

// Configure project-specific tweaks and to-dos.
configure(project(":lucene:analysis:common")) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing javadocs
    javadocMissingLevel = "class"
  }
}

configure([
    project(":lucene:analysis:kuromoji"),
    project(":lucene:analysis:nori"),
    project(":lucene:analysis:opennlp"),
    project(":lucene:analysis:smartcn"),
    project(":lucene:benchmark"),
    project(":lucene:codecs"),
    project(":lucene:grouping"),
    project(":lucene:highlighter"),
    project(":lucene:luke"),
    project(":lucene:monitor"),
    project(":lucene:queries"),
    project(":lucene:queryparser"),
    project(":lucene:replicator"),
    project(":lucene:spatial-extras"),
]) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing javadocs
    javadocMissingLevel = "class"
  }
}

configure([
    project(":lucene:analysis:icu"),
    project(":lucene:analysis:morfologik"),
    project(":lucene:analysis:phonetic"),
    project(":lucene:analysis:stempel"),
    project(":lucene:classification"),
    project(":lucene:demo"),
    project(":lucene:expressions"),
    project(":lucene:facet"),
    project(":lucene:join"),
    project(":lucene:spatial3d"),
    project(":lucene:suggest"),
]) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing @param tags
    javadocMissingLevel = "method"
  }
}

configure(project(":lucene:backward-codecs")) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing @param tags
    javadocMissingLevel = "method"
  }
}

configure(project(":lucene:test-framework")) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing javadocs
    javadocMissingLevel = "class"
    // TODO: clean up split packages
    javadocMissingIgnore = [
        "org.apache.lucene.analysis",
        "org.apache.lucene.analysis.standard",
        "org.apache.lucene.codecs",
        "org.apache.lucene.codecs.blockterms",
        "org.apache.lucene.codecs.bloom",
        "org.apache.lucene.codecs.compressing",
        "org.apache.lucene.codecs.uniformsplit",
        "org.apache.lucene.codecs.uniformsplit.sharedterms",
        "org.apache.lucene.geo",
        "org.apache.lucene.index",
        "org.apache.lucene.search",
        "org.apache.lucene.search.similarities",
        "org.apache.lucene.search.spans",
        "org.apache.lucene.store",
        "org.apache.lucene.util",
        "org.apache.lucene.util.automaton",
        "org.apache.lucene.util.fst"
    ]
  }
}

configure(project(":lucene:sandbox")) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing javadocs
    javadocMissingLevel = "class"
  }
}

configure(project(":lucene:misc")) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing javadocs
    javadocMissingLevel = "class"
  }
}

configure(project(":lucene:core")) {
  project.tasks.withType(RenderJavadocTask) {
    // TODO: fix missing javadocs
    javadocMissingLevel = "class"
    // some packages are fixed already
    javadocMissingMethod = [
        "org.apache.lucene.util.automaton",
        "org.apache.lucene.analysis.standard",
        "org.apache.lucene.analysis.tokenattributes",
        "org.apache.lucene.document",
        "org.apache.lucene.search.similarities",
        "org.apache.lucene.index",
        "org.apache.lucene.codecs",
        "org.apache.lucene.codecs.lucene50",
        "org.apache.lucene.codecs.lucene60",
        "org.apache.lucene.codecs.lucene80",
        "org.apache.lucene.codecs.lucene84",
        "org.apache.lucene.codecs.lucene86",
        "org.apache.lucene.codecs.lucene87",
        "org.apache.lucene.codecs.perfield"
    ]
  }
}

// Fix for Java 11 Javadoc tool that cannot handle split packages between modules correctly.
// (by removing all the packages which are part of lucene-core)
// See: https://issues.apache.org/jira/browse/LUCENE-8738?focusedCommentId=16818106&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16818106
// LUCENE-9499: This workaround should be applied only to test-framework (we have no split package in other modules).
configure(project(":lucene:test-framework")) {
  project.tasks.withType(RenderJavadocTask) {
    doLast {
      Set luceneCorePackages = file("${project(':lucene:core').tasks[name].outputDir}/element-list").readLines('UTF-8').toSet();
      File elementFile = file("${outputDir}/element-list");
      List elements = elementFile.readLines('UTF-8');
      elements.removeAll(luceneCorePackages)
      elementFile.write(elements.join('\n').concat('\n'), 'UTF-8');
    }
  }
}

configure(project(':lucene:demo')) {
  project.tasks.withType(RenderJavadocTask) {
    // For the demo, we link the example source in the javadocs, as it's ref'ed elsewhere
    linksource = true
  }
}

// Add cross-project documentation task dependencies:
// - each RenderJavaDocs task gets a dependency to all tasks with the same name in its dependencies
// - the dependency is using dependsOn with a closure to enable lazy evaluation
configure(subprojects) {
  project.tasks.withType(RenderJavadocTask) { task ->
    task.dependsOn {
      task.project.configurations.implementation.allDependencies.withType(ProjectDependency).collect { dep ->
        def otherProject = dep.dependencyProject
        return otherProject.tasks.findByName(task.name)
      }
    }
  }
}

class RenderJavadocTask extends DefaultTask {
  @InputFiles
  @SkipWhenEmpty
  SourceDirectorySet srcDirSet;
  
  @OutputDirectory
  File outputDir

  @CompileClasspath
  FileCollection classpath

  @CompileClasspath
  FileCollection docletpath

  @Input
  String title

  @Input
  boolean linksource = false

  @Input
  boolean relativeProjectLinks = false

  @Input
  def offlineLinks = [:]

  @Input
  @Optional
  Property<String> luceneDocUrl = project.objects.property(String)

  // default is to require full javadocs
  @Input
  String javadocMissingLevel = "parameter"

  // anything in these packages is checked with level=method. This allows iteratively fixing one package at a time.
  @Input
  List<String> javadocMissingMethod = []

  // default is not to ignore any elements, should only be used to workaround split packages
  @Input
  List<String> javadocMissingIgnore = []

  @Nullable
  @Optional
  @Input
  def executable

  @Input
  def taskResources

  /** Utility method to recursively collect all tasks with same name like this one that we depend on */
  private Set findRenderTasksInDependencies() {
    Set found = []
    def collectDeps
    collectDeps = { task -> task.taskDependencies.getDependencies(task).findAll{ it.name == this.name && it.enabled && !found.contains(it) }.each{
      found << it
      collectDeps(it)
    }}
    collectDeps(this)
    return found
  }

  @TaskAction
  public void render() {
    def srcDirs = srcDirSet.srcDirs.findAll { dir -> dir.exists() }
    def optionsFile = project.file("${getTemporaryDir()}/javadoc-options.txt")
    
    // create the directory, so relative link calculation knows that it's a directory:
    outputDir.mkdirs();
    
    def opts = []
    opts << [ '-overview', project.file("${srcDirs[0]}/overview.html") ]
    opts << [ '-sourcepath', srcDirs.join(File.pathSeparator) ]
    opts << [ '-subpackages', project.path.startsWith(':lucene') ? 'org.apache.lucene' : 'org.apache.solr' ]
    opts << [ '-d', outputDir ]
    opts << '-protected'
    opts << [ '-encoding', 'UTF-8' ]
    opts << [ '-charset', 'UTF-8' ]
    opts << [ '-docencoding', 'UTF-8' ]
    opts << '-noindex'
    opts << '-author'
    opts << '-version'
    if (linksource) {
      opts << '-linksource'
    }
    opts << '-use'
    opts << [ '-locale', 'en_US' ]
    opts << [ '-windowtitle', title ]
    opts << [ '-doctitle', title ]
    if (!classpath.isEmpty()) {
      opts << [ '-classpath', classpath.asPath ]
    }
    opts << [ '-bottom', "<i>Copyright &copy; 2000-${project.buildYear} Apache Software Foundation. All Rights Reserved.</i>" ]

    opts << [ '-tag', 'lucene.experimental:a:WARNING: This API is experimental and might change in incompatible ways in the next release.' ]
    opts << [ '-tag', 'lucene.internal:a:NOTE: This API is for internal purposes only and might change in incompatible ways in the next release.' ]
    opts << [ '-tag', "lucene.spi:t:SPI Name (case-insensitive: if the name is 'htmlStrip', 'htmlstrip' can be used when looking up the service)." ]

    opts << [ '-doclet', "org.apache.lucene.missingdoclet.MissingDoclet" ]
    opts << [ '-docletpath', docletpath.asPath ]
    opts << [ '--missing-level', javadocMissingLevel ]
    if (javadocMissingIgnore) {
      opts << [ '--missing-ignore', String.join(',', javadocMissingIgnore) ]
    }
    if (javadocMissingMethod) {
      opts << [ '--missing-method', String.join(',', javadocMissingMethod) ]
    }

    opts << [ '-quiet' ]

    def allOfflineLinks = [:]
    allOfflineLinks.putAll(offlineLinks)

    // Resolve inter-project links:
    // - find all (enabled) tasks this tasks depends on (with same name), calling findRenderTasksInDependencies()
    // - sort the tasks preferring those whose project name equals 'core', then lexigraphical by path
    // - for each task get output dir to create relative or absolute link
    // NOTE: explicitly exclude solr/test-framework, or attempting to link to lucene-test-framework because if we did javadoc would
    // attempt to link class refs in in org.apache.lucene, causing broken links. (either broken links to things like "Directory" if
    // lucene-test-framework was first, or broken links to things like LuceneTestCase if lucene-core was first)
    findRenderTasksInDependencies()
        .sort(false, Comparator.comparing { (it.project.name != 'core') as Boolean }.thenComparing(Comparator.comparing { it.path }))
        .each { otherTask ->
          def otherProject = otherTask.project
          // For relative links we compute the actual relative link between projects.
          if (relativeProjectLinks) {
            def pathTo = otherTask.outputDir.toPath().toAbsolutePath()
            def pathFrom = outputDir.toPath().toAbsolutePath()
            def relative = pathFrom.relativize(pathTo).toString().replace(File.separator, '/')
            opts << ['-link', relative]
          } else {
            // For absolute links, we determine the target URL by assembling the full URL (if base is available).
            def value = luceneDocUrl.getOrElse(null)
            if (value) {
              allOfflineLinks.put("${value}/${otherProject.relativeDocPath}/".toString(), otherTask.outputDir)
            }
          }
        }

    // Add offline links.
    allOfflineLinks.each { url, dir ->
      // Some sanity check/ validation here to ensure dir/package-list or dir/element-list is present.
      if (!project.file("$dir/package-list").exists() &&
          !project.file("$dir/element-list").exists()) {
        throw new GradleException("Expected pre-rendered package-list or element-list at ${dir}.")
      }
      logger.info("Linking ${url} to ${dir}")
      opts << [ '-linkoffline', url, dir ]
    }

    opts << [ '--release', 11 ]
    opts << '-Xdoclint:all,-missing'

    // Temporary file that holds all javadoc options for the current task.
    optionsFile.withWriter("UTF-8", { writer ->
      // escapes an option with single quotes or whitespace to be passed in the options.txt file for
      def escapeJavadocOption = { String s -> (s =~ /[ '"]/) ? ("'" + s.replaceAll(/[\\'"]/, /\\$0/) + "'") : s }

      opts.each { entry ->
        if (entry instanceof List) {
          writer.write(entry.collect { escapeJavadocOption(it as String) }.join(" "))
        } else {
          writer.write(escapeJavadocOption(entry as String))
        }
        writer.write('\n')
      }
    })

    def javadocCmd = {
      if (executable == null) {
        JavaInstallationRegistry registry = project.extensions.getByType(JavaInstallationRegistry)
        JavaInstallation currentJvm = registry.installationForCurrentVirtualMachine.get()
        return currentJvm.jdk.get().javadocExecutable.asFile
      } else {
        return project.file(executable)
      }
    }()

    logger.info("Javadoc executable used: ${javadocCmd}")

    def outputFile = project.file("${getTemporaryDir()}/javadoc-output.txt")
    def result

    outputFile.withOutputStream { output ->
      result = project.exec {
        executable javadocCmd

        // we want to capture both stdout and stderr to the same
        // stream but gradle attempts to close these separately
        // (it has two independent pumping threads) and it can happen
        // that one still tries to write something when the other closed
        // the underlying output stream.
        def wrapped = new java.io.FilterOutputStream(output) {
          public void close() { 
            // no-op. we close this stream manually.
          }
        }
        standardOutput = wrapped
        errorOutput = wrapped

        args += [ "@${optionsFile}" ]

        // -J flags can't be passed via options file... (an error "javadoc: error - invalid flag: -J-Xmx512m" occurs.)
        args += [ "-J-Xmx512m" ]
        // force locale to be "en_US" (fix for: https://bugs.openjdk.java.net/browse/JDK-8222793)
        args += [ "-J-Duser.language=en", "-J-Duser.country=US" ]

        ignoreExitValue true
      }
    }

    if (result.getExitValue() != 0) {
      // Pipe the output to console. Intentionally skips any encoding conversion 
      // and pumps raw bytes.
      System.out.write(outputFile.bytes)
      System.out.flush()

      def cause
      try {
        result.rethrowFailure()
      } catch (ex) {
        cause = ex
      }
      throw new GradleException("Javadoc generation failed for ${project.path},\n  Options file at: ${optionsFile}\n  Command output at: ${outputFile}", cause)
    }

    // append some special table css, prettify css
    ant.concat(destfile: "${outputDir}/stylesheet.css", append: "true", fixlastline: "true", encoding: "UTF-8") {
      filelist(dir: taskResources, files: "table_padding.css")
      filelist(dir: project.file("${taskResources}/prettify"), files: "prettify.css")
    }

    // append prettify to scripts
    ant.concat(destfile: "${outputDir}/script.js", append: "true", fixlastline: "true", encoding: "UTF-8") {
      filelist(dir: project.file("${taskResources}/prettify"), files: "prettify.js inject-javadocs.js")
    }

    ant.fixcrlf(srcdir: outputDir, includes: "stylesheet.css script.js", eol: "lf", fixlast: "true", encoding: "UTF-8")
  }
}