HADOOP-12135. cleanup releasedocmaker

dev-support/releasedocmaker.py

@@ -19,6 +19,7 @@
 from glob import glob
 from optparse import OptionParser
 from time import gmtime, strftime
+import pprint
 import os
 import re
 import sys
@@ -99,23 +100,44 @@ def mstr(obj):
     return ""
   return unicode(obj)
 
-def buildindex(master):
+def buildindex(title,license):
   versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
   with open("index.md","w") as indexfile:
+    if license is True:
+      indexfile.write(asflicense)
     for v in versions:
-      indexfile.write("* Apache Hadoop v%s\n" % (v))
+      indexfile.write("* %s v%s\n" % (title,v))
       for k in ("Changes","Release Notes"):
-        indexfile.write("    *  %s\n" %(k))
+        indexfile.write("    * %s (%s/%s.%s.html)\n" \
-        indexfile.write("        * [Combined %s](%s/%s.%s.html)\n" \
           % (k,v,k.upper().replace(" ",""),v))
-        if not master:
-          indexfile.write("        * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \
-            % (k,v,k.upper().replace(" ",""),v))
-          for p in ("HDFS","MapReduce","YARN"):
-            indexfile.write("        * [%s %s](%s/%s.%s.%s.html)\n" \
-              % (p,k,v,k.upper().replace(" ",""),p.upper(),v))
   indexfile.close()
 
+class GetVersions:
+  """ yo """
+  def __init__(self,versions, projects):
+    versions = versions
+    projects = projects
+    self.newversions = []
+    pp = pprint.PrettyPrinter(indent=4)
+    at=0
+    end=1
+    count=100
+    versions.sort()
+    print "Looking for %s through %s"%(versions[0],versions[-1])
+    for p in projects:
+      resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
+      data = json.loads(resp.read())
+      for d in data:
+        if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
+          print "Adding %s to the list" % d['name']
+          self.newversions.append(d['name'])
+    newlist=list(set(self.newversions))
+    self.newversions=newlist
+
+  def getlist(self):
+      pp = pprint.PrettyPrinter(indent=4)
+      return(self.newversions)
+
 class Version:
   """Represents a version number"""
   def __init__(self, data):
@@ -261,8 +283,10 @@ class Jira:
 class JiraIter:
   """An Iterator of JIRAs"""
 
-  def __init__(self, versions):
+  def __init__(self, version, projects):
-    self.versions = versions
+    self.version = version
+    self.projects = projects
+    v=str(version).replace("-SNAPSHOT","")
 
     resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
     data = json.loads(resp.read())
@@ -276,7 +300,7 @@ class JiraIter:
     end=1
     count=100
     while (at < end):
-      params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
+      params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
       resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
       data = json.loads(resp.read())
       if (data.has_key('errorMessages')):
@@ -286,8 +310,6 @@ class JiraIter:
       self.jiras.extend(data['issues'])
 
       needaversion=False
-      for j in versions:
-        v=str(j).replace("-SNAPSHOT","")
       if v not in releaseVersion:
         needaversion=True
 
@@ -351,21 +373,29 @@ class Outputs:
       self.writeKeyRaw(jira.getProject(), line)
 
 def main():
-  parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]",
+  parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
 		epilog=
                "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
                " named after the highest version provided.")
+  parser.add_option("-i","--index", dest="index", action="store_true",
+             default=False, help="build an index file")
+  parser.add_option("-l","--license", dest="license", action="store_false",
+             default=True, help="Add an ASF license")
+  parser.add_option("-n","--lint", dest="lint", action="store_true",
+             help="use lint flag to exit on failures")
+  parser.add_option("-p", "--project", dest="projects",
+             action="append", type="string",
+             help="projects in JIRA to include in releasenotes", metavar="PROJECT")
+  parser.add_option("-r", "--range", dest="range", action="store_true",
+             default=False, help="Given versions are a range")
+  parser.add_option("-t", "--projecttitle", dest="title",
+             type="string",
+             help="Title to use for the project (default is Apache PROJECT)")
+  parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
+             default=False, help="use current date for unreleased versions")
   parser.add_option("-v", "--version", dest="versions",
              action="append", type="string",
              help="versions in JIRA to include in releasenotes", metavar="VERSION")
-  parser.add_option("-m","--master", dest="master", action="store_true",
-             help="only create the master, merged project files")
-  parser.add_option("-i","--index", dest="index", action="store_true",
-             help="build an index file")
-  parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
-             help="use current date for unreleased versions")
-  parser.add_option("-n","--lint", dest="lint", action="store_true",
-             help="use lint flag to exit on failures")
   (options, args) = parser.parse_args()
 
   if (options.versions is None):
@@ -377,52 +407,52 @@ def main():
   if (len(options.versions) <= 0):
     parser.error("At least one version needs to be supplied")
 
-  versions = [ Version(v) for v in options.versions ];
+  projects = options.projects
+
+  if (options.range is True):
+    versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
+  else:
+    versions = [ Version(v) for v in options.versions ]
   versions.sort();
 
-  maxVersion = str(versions[-1])
+  if (options.title is None):
+    title=projects[0]
+  else:
+    title=options.title
 
-  jlist = JiraIter(versions)
+  for v in versions:
-  version = maxVersion
+    vstr=str(v)
+    jlist = JiraIter(vstr,projects)
 
-  if version in releaseVersion:
+    if vstr in releaseVersion:
-    reldate=releaseVersion[version]
+      reldate=releaseVersion[vstr]
     elif options.usetoday:
       reldate=strftime("%Y-%m-%d", gmtime())
     else:
       reldate="Unreleased"
 
-  if not os.path.exists(version):
+    if not os.path.exists(vstr):
-    os.mkdir(version)
+      os.mkdir(vstr)
 
-  if options.master:
     reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
       "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
-      [], {"ver":maxVersion, "date":reldate})
+      [], {"ver":v, "date":reldate, "title":title})
     choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
       "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
-      [], {"ver":maxVersion, "date":reldate})
+      [], {"ver":v, "date":reldate, "title":title})
-  else:
-    reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
-      "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
-      ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
-    choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
-      "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
-      ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
 
+    if (options.license is True):
       reloutputs.writeAll(asflicense)
       choutputs.writeAll(asflicense)
 
-  relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
+    relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
       'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
-
+    chhead = '# %(title)s Changelog\n\n' \
-  chhead = '# Hadoop Changelog\n\n' \
       '## Release %(ver)s - %(date)s\n'\
       '\n'
 
     reloutputs.writeAll(relhead)
     choutputs.writeAll(chhead)
-
     errorCount=0
     warningCount=0
     lintMessage=""
@@ -539,7 +569,7 @@ def main():
     choutputs.close()
 
   if options.index:
-    buildindex(options.master)
+    buildindex(title,options.license)
 
 if __name__ == "__main__":
   main()