LUCENE-3887: fix smoke tester to fail when package.html is missing or public classes don't have description in package-summary

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1303830 13f79535-47bb-0310-9956-ffa450edef68

dev-tools/scripts/checkJavaDocs.py

@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+import re
+
+reHREF = re.compile('<a.*?>(.*?)</a>', re.IGNORECASE)
+
+reMarkup = re.compile('<.*?>')
+
+def checkSummary(fullPath):
+  printed = False
+  f = open(fullPath)
+  lastLine = None
+  anyMissing = False
+  sawPackage = False
+  desc = []
+  for line in f.readlines():
+    lineLower = line.strip().lower()
+    if desc is not None:
+      # TODO: also detect missing description in overview-summary
+      if lineLower.startswith('package ') or lineLower.startswith('<h1 title="package" '):
+        sawPackage = True
+      elif sawPackage:
+        if lineLower.startswith('<table ') or lineLower.startswith('<b>see: '):
+          desc = ' '.join(desc)
+          desc = reMarkup.sub(' ', desc)
+          desc = desc.strip()
+          if desc == '':
+            if not printed:
+              print
+              print fullPath
+              printed = True
+            print '  no package description (missing package.html in src?)'
+            anyMissing = True
+          desc = None
+        else:
+          desc.append(lineLower)
+      
+    if lineLower in ('<td>&nbsp;</td>', '<td></td>', '<td class="collast">&nbsp;</td>'):
+      m = reHREF.search(lastLine)
+      if not printed:
+        print
+        print fullPath
+        printed = True
+      print '  missing: %s' % unescapeHTML(m.group(1))
+      anyMissing = True
+    lastLine = line
+  if desc is not None and fullPath.find('/overview-summary.html') == -1:
+    raise RuntimeError('BUG: failed to locate description in %s' % fullPath)
+  f.close()
+  return anyMissing
+
+def unescapeHTML(s):
+  s = s.replace('&lt;', '<')
+  s = s.replace('&gt;', '>')
+  s = s.replace('&amp;', '&')
+  return s
+
+def checkPackageSummaries(root):
+  """
+  Just checks for blank summary lines in package-summary.html; returns
+  True if there are problems.
+  """
+  
+  #for dirPath, dirNames, fileNames in os.walk('%s/lucene/build/docs/api' % root):
+
+  if False:
+    os.chdir(root)
+    print
+    print 'Run "ant javadocs" > javadocs.log...'
+    if os.system('ant javadocs > javadocs.log 2>&1'):
+      print '  FAILED'
+      sys.exit(1)
+    
+  anyMissing = False
+  for dirPath, dirNames, fileNames in os.walk(root):
+    
+    if dirPath.find('/all/') != -1:
+      # These are dups (this is a bit risk, eg, root IS this /all/ directory..)
+      continue
+
+    if 'package-summary.html' in fileNames:
+      if checkSummary('%s/package-summary.html' % dirPath):
+        anyMissing = True
+    if 'overview-summary.html' in fileNames:        
+      if checkSummary('%s/overview-summary.html' % dirPath):
+        anyMissing = True
+
+  return anyMissing
+
+if __name__ == '__main__':
+  checkPackageSummaries(sys.argv[1])

dev-tools/scripts/smokeTestRelease.py

@@ -26,6 +26,7 @@ from collections import defaultdict
 import xml.etree.ElementTree as ET
 import filecmp
 import platform
+import checkJavaDocs
 
 # This tool expects to find /lucene and /solr off the base URL.  You
 # must have a working gpg, tar, unzip in your path.  This has been
@@ -243,7 +244,7 @@ def testChangesText(dir, version, project):
     # NOTE: O(N) but N should be smallish:
     if 'CHANGES.txt' in files:
       fullPath = '%s/CHANGES.txt' % root
-      print 'CHECK %s' % fullPath
+      #print 'CHECK %s' % fullPath
       checkChangesContent(open(fullPath).read(), version, fullPath, project, False)
       
 def checkChangesContent(s, version, name, project, isHTML):
@@ -383,6 +384,9 @@ def verifyUnpacked(project, artifact, unpackPath, version):
       # test javadocs
       print '    generate javadocs w/ Java 5...'
       run('export JAVA_HOME=%s; ant javadocs' % JAVA5_HOME, '%s/javadocs.log' % unpackPath)
+      if checkJavaDocs.checkPackageSummaries('build/docs/api'):
+        raise RuntimeError('javadoc summaries failed')
+      
     else:
       print '    run tests w/ Java 6...'
       run('export JAVA_HOME=%s; ant test' % JAVA6_HOME, '%s/test.log' % unpackPath)
@@ -421,6 +425,21 @@ def verifyUnpacked(project, artifact, unpackPath, version):
 
   testChangesText('.', version, project)
 
+  if project == 'lucene' and not isSrc:
+    print '    check Lucene\'s javadoc JAR'
+    unpackJavadocsJar('%s/lucene-core-%s-javadoc.jar' % (unpackPath, version), unpackPath)
+
+def unpackJavadocsJar(jarPath, unpackPath):
+  destDir = '%s/javadocs' % unpackPath
+  if os.path.exists(destDir):
+    shutil.rmtree(destDir)
+  os.makedirs(destDir)
+  os.chdir(destDir)
+  run('unzip %s' % jarPath, '%s/unzip.log' % destDir)
+  if checkJavaDocs.checkPackageSummaries('.'):
+    raise RuntimeError('javadoc problems')
+  os.chdir(unpackPath)
+
 def testDemo(isSrc, version):
   print '    test demo...'
   sep = ';' if platform.system().lower().startswith('cygwin') else ':'