LUCENE-9242: generate javadocs by calling Ant javadoc task (#1304)

build.gradle

@@ -65,6 +65,7 @@ apply from: file('gradle/ant-compat/folder-layout.gradle')
 apply from: file('gradle/defaults.gradle')
 apply from: file('gradle/defaults-java.gradle')
 apply from: file('gradle/defaults-javadoc.gradle')
+apply from: file('gradle/render-javadoc.gradle')
 apply from: file('gradle/testing/defaults-tests.gradle')
 apply from: file('gradle/testing/randomization.gradle')
 apply from: file('gradle/testing/fail-on-no-tests.gradle')

gradle/render-javadoc.gradle

@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// generate javadocs by using Ant javadoc task
+
+allprojects {
+  plugins.withType(JavaPlugin) {
+    ext {
+      javadocRoot = project.path.startsWith(':lucene') ? project(':lucene').file("build/docs") : project(':solr').file("build/docs")
+      javadocDestDir = "${javadocRoot}/${project.name}"
+    }
+
+    task renderJavadoc {
+      description "Generates Javadoc API documentation for the main source code. This invokes Ant Javadoc Task."
+      group "documentation"
+
+      ext {
+        linksource = "no"
+        linkJUnit = false
+        linkHref = []
+      }
+
+      dependsOn sourceSets.main.compileClasspath
+
+      inputs.files { sourceSets.main.java.asFileTree }
+      outputs.dir project.javadocRoot
+
+      def libName = project.path.startsWith(":lucene") ? "Lucene" : "Solr"
+      def title = "${libName} ${project.version} ${project.name} API".toString()
+
+      doFirst {
+        def srcDirs = sourceSets.main.java.srcDirs.findAll { dir -> dir.exists() }
+
+        ant.javadoc(
+            overview: file("src/java/overview.html"),
+            packagenames: "org.apache.lucene.*,org.apache.solr.*",
+            destDir: javadocDestDir,
+            access: "protected",
+            encoding: "UTF-8",
+            charset: "UTF-8",
+            docencoding: "UTF-8",
+            noindex: "true",
+            includenosourcepackages: "true",
+            author: "true",
+            version: "true",
+            linksource: linksource,
+            use: "true",
+            failonerror: "true",
+            locale: "en_US",
+            windowtitle: title,
+            doctitle: title,
+            maxmemory: "512m",
+            classpath: sourceSets.main.compileClasspath.asPath,
+            bottom: "<i>Copyright &copy; 2000-${buildYear} Apache Software Foundation. All Rights Reserved.</i>"
+        ) {
+          srcDirs.collect { srcDir ->
+            packageset(dir: srcDir)
+          }
+
+          tag(name: "lucene.experimental", description: "WARNING: This API is experimental and might change in incompatible ways in the next release.")
+          tag(name: "lucene.internal", description: "NOTE: This API is for internal purposes only and might change in incompatible ways in the next release.")
+          tag(name: "lucene.spi", description: "SPI Name (Note: This is case-insensitive. e.g., if the name is 'htmlStrip', 'htmlstrip' can be used when looking up the service):", scope: "types")
+
+          // resolve links to JavaSE and JUnit API
+          link(offline: "true", href: "https://docs.oracle.com/en/java/javase/11/docs/api/", packageListLoc: project(":lucene").file("tools/javadoc/java11/").toString())
+          if (linkJUnit) {
+            link(offline: "true", href: "https://junit.org/junit4/javadoc/4.12/", packageListLoc: project(":lucene").file("tools/javadoc/junit").toString())
+          }
+          // resolve inter-module links if 'linkHref' property is specified
+          linkHref.collect { path ->
+            link(href: path)
+          }
+
+          arg(line: "--release 11")
+          arg(line: "-Xdoclint:all,-missing")
+
+          // force locale to be "en_US" (fix for: https://bugs.openjdk.java.net/browse/JDK-8222793)
+          arg(line: "-J-Duser.language=en -J-Duser.country=US")
+        }
+
+        // append some special table css, prettify css
+        ant.concat(destfile: "${javadocDestDir}/stylesheet.css", append: "true", fixlastline: "true", encoding: "UTF-8") {
+          filelist(dir: project(":lucene").file("tools/javadoc"), files: "table_padding.css")
+          filelist(dir: project(":lucene").file("tools/prettify"), files: "prettify.css")
+        }
+        // append prettify to scripts
+        ant.concat(destfile: "${javadocDestDir}/script.js", append: "true", fixlastline: "true", encoding: "UTF-8") {
+          filelist(dir: project(':lucene').file("tools/prettify"), files: "prettify.js inject-javadocs.js")
+        }
+        ant.fixcrlf(srcdir: javadocDestDir, includes: "stylesheet.css script.js", eol: "lf", fixlast: "true", encoding: "UTF-8")
+      }
+    }
+  }
+}
+
+configure(subprojects.findAll { it.path.startsWith(':lucene') && it.path != ':lucene:core' }) {
+
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:core:renderJavadoc'
+      linkHref += [ "../core" ]
+
+      doLast {
+        // fix for Java 11 Javadoc tool that cannot handle split packages between modules correctly (by removing all the packages which are part of lucene-core)
+        // problem description: [https://issues.apache.org/jira/browse/LUCENE-8738?focusedCommentId=16818106&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16818106]
+        ant.local(name: "element-list-regex") // contains a regex for all package names which are in lucene-core's javadoc
+        ant.loadfile(property: "element-list-regex", srcFile: "${project.javadocRoot}/core/element-list", encoding: "utf-8") {
+          filterchain {
+            tokenfilter(delimoutput: "|") {
+              replacestring(from: ".", to: "\\.")
+            }
+          }
+        }
+        ant.replaceregexp(
+            encoding: "UTF-8",
+            file: "${project.javadocDestDir}/element-list",
+            byline: "true",
+            match: "^(\${element-list-regex})\$",
+            replace: "")
+      }
+    }
+  }
+}
+
+configure(subprojects.findAll { it.path.startsWith(':lucene:analysis') }) {
+  plugins.withType(JavaPlugin) {
+    ext {
+      javadocDestDir = "${javadocRoot}/analyzers-${project.name}"
+    }
+
+    renderJavadoc {
+      if (project.path != ':lucene:analysis:common') {
+        dependsOn ':lucene:analysis:common:renderJavadoc'
+        linkHref += [ "../analyzers-common" ]
+      }
+    }
+  }
+}
+
+configure(project(':lucene:benchmark')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      [':lucene:memory',
+       ':lucene:highlighter',
+       ':lucene:analysis:common',
+       ':lucene:queryparser',
+       ':lucene:facet',
+       ':lucene:spatial-extras'].collect { path ->
+        dependsOn "${path}:renderJavadoc"
+      }
+
+      linkHref += [ '../memory', '../highlighter', '../analyzers-common', '../queryparser', '../facet', '../spatial-extras' ]
+    }
+  }
+}
+
+configure(project(':lucene:classification')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      [':lucene:queries', ':lucene:analysis:common', ':lucene:grouping'].collect { path ->
+        dependsOn "${path}:renderJavadoc"
+      }
+
+      linkHref += ['../queries', '../analyzers-common', '../grouping']
+    }
+  }
+}
+
+configure(project(':lucene:demo')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      [':lucene:analysis:common',
+       ':lucene:queryparser',
+       ':lucene:queries',
+       ':lucene:facet',
+       ':lucene:expressions'].collect { path ->
+        dependsOn "${path}:renderJavadoc"
+      }
+      linkHref += ['../analyzers-common', '../queryparser', '../queries', '../facet', '../expressions']
+
+      // we link the example source in the javadocs, as it's ref'ed elsewhere
+      linksource = "yes"
+    }
+  }
+}
+
+configure(project(':lucene:grouping')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:queries:renderJavadoc'
+      linkHref += [ '../queries' ]
+    }
+  }
+}
+
+configure(project(':lucene:highlighter')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:memory:renderJavadoc'
+      linkHref += [ '../memory' ]
+    }
+  }
+}
+
+configure(project(':lucene:monitor')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      [':lucene:memory', ':lucene:analysis:common', ':lucene:queryparser'].collect { path ->
+        dependsOn "${path}:renderJavadoc"
+      }
+      linkHref += [ '../memory', '../analyzers-common', '../queryparser' ]
+    }
+  }
+}
+
+configure(project(':lucene:queryparser')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      [':lucene:queries', ':lucene:sandbox'].collect { path ->
+        dependsOn "${path}:renderJavadoc"
+      }
+      linkHref += [ '../queries', '../sandbox' ]
+    }
+  }
+}
+
+configure(project(':lucene:replicator')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:facet:renderJavadoc'
+      linkHref += [ '../facet' ]
+    }
+  }
+}
+
+configure(project(':lucene:spatial-extras')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:spatial3d:renderJavadoc'
+      linkHref += [ '../spatial3d' ]
+    }
+  }
+}
+
+configure(project(':lucene:suggest')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:analysis:common:renderJavadoc'
+      linkHref += [ '../analyzers-common' ]
+    }
+  }
+}
+
+configure(project(':lucene:test-framework')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':lucene:codecs:renderJavadoc'
+      linkJUnit = true
+      linkHref += [ '../codecs' ]
+    }
+  }
+}
+
+configure(subprojects.findAll { it.path.startsWith(':solr') }) {
+  plugins.withType(JavaPlugin) {
+    ext {
+      javadocDestDir = "${javadocRoot}/solr-${project.name}"
+    }
+
+    def hasJavdocsTask = project.tasks.collect { it.name }.contains('renderJavadoc')
+    if (hasJavdocsTask) {
+      renderJavadoc {
+        // TODO: generate links to lucene modules. i.e., port "solr-invoke-javadoc" Ant macro.
+      }
+    }
+  }
+}
+
+configure(project(':solr:core')) {
+  plugins.withType(JavaPlugin) {
+    // specialized to ONLY depend on solrj
+    renderJavadoc {
+      dependsOn ':solr:solrj:renderJavadoc'
+      linkHref += [ '../solr-solrj' ]
+    }
+  }
+}
+
+configure(subprojects.findAll { it.path.startsWith(':solr:contrib') }) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':solr:solrj:renderJavadoc'
+      dependsOn ':solr:core:renderJavadoc'
+      linkHref += [ '../solr-solrj', '../solr-core' ]
+    }
+  }
+}
+
+configure(project(':solr:contrib:dataimporthandler-extras')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      dependsOn ':solr:contrib:dataimporthandler:renderJavadoc'
+      linkHref += [ '../solr-dataimporthandler' ]
+    }
+  }
+}
+
+configure(project(':solr:contrib:extraction')) {
+  plugins.withType(JavaPlugin) {
+    ext {
+      javadocDestDir = "${javadocRoot}/solr-cell"
+    }
+  }
+}
+
+configure(project(':solr:test-framework')) {
+  plugins.withType(JavaPlugin) {
+    renderJavadoc {
+      linkJUnit = true
+    }
+  }
+}
+
+
+configure(subprojects.findAll { it.path in [':solr:solr-ref-guide', ':solr:server', ':solr:webapp']}) {
+  afterEvaluate {
+    project.tasks.findByPath("renderJavadoc").enabled = false
+  }
+}

gradle/validation/missing-docs-check.gradle

@@ -34,8 +34,8 @@ allprojects {
         ':lucene:spatial3d',
     ]
 
-    task checkMissingDocsDefault(type: CheckMissingDocsTask, dependsOn: 'javadoc') {
+    task checkMissingDocsDefault(type: CheckMissingDocsTask, dependsOn: 'renderJavadoc') {
-      dirs += [ project.javadoc.destinationDir ]
+      dirs += [ project.file(project.javadocDestDir) ]
 
       // TODO: add missing docs for all classes and bump this to level=class
       if (project.path.startsWith(":solr")) {
@@ -59,7 +59,7 @@ allprojects {
 configure(project(':lucene:core')) {
   // Defer until java plugin has been applied, otherwise we can't resolve project.javadoc.
   plugins.withType(JavaPlugin) {
-    task checkMissingDocsMethod(type: CheckMissingDocsTask, dependsOn: 'javadoc') {
+    task checkMissingDocsMethod(type: CheckMissingDocsTask, dependsOn: 'renderJavadoc') {
       level = 'method'
     }
 
@@ -71,7 +71,7 @@ configure(project(':lucene:core')) {
         "org/apache/lucene/search/similarities",
         "org/apache/lucene/index",
         "org/apache/lucene/codecs"
-    ].collect { path -> file("${project.javadoc.destinationDir}/${path}") }
+    ].collect { path -> file("${project.javadocDestDir}/${path}") }
 
     checkMissingDocs {
       dependsOn checkMissingDocsMethod

gradle/validation/precommit.gradle

@@ -40,8 +40,7 @@ configure(rootProject) {
           "javadoc",
           "rat",
           "ecjLint",
-          // FIXME: enable this line once Javadocs are correctly generated
+          "checkMissingDocs"
-          // "checkMissingDocs"
       ]}
     }
   }