diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py index 8e68b3cb9e0..6e0126073c0 100755 --- a/dev-support/releasedocmaker.py +++ b/dev-support/releasedocmaker.py @@ -19,6 +19,7 @@ from glob import glob from optparse import OptionParser from time import gmtime, strftime +import pprint import os import re import sys @@ -99,23 +100,44 @@ def mstr(obj): return "" return unicode(obj) -def buildindex(master): +def buildindex(title,license): versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*"))) with open("index.md","w") as indexfile: + if license is True: + indexfile.write(asflicense) for v in versions: - indexfile.write("* Apache Hadoop v%s\n" % (v)) + indexfile.write("* %s v%s\n" % (title,v)) for k in ("Changes","Release Notes"): - indexfile.write(" * %s\n" %(k)) - indexfile.write(" * [Combined %s](%s/%s.%s.html)\n" \ + indexfile.write(" * %s (%s/%s.%s.html)\n" \ % (k,v,k.upper().replace(" ",""),v)) - if not master: - indexfile.write(" * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \ - % (k,v,k.upper().replace(" ",""),v)) - for p in ("HDFS","MapReduce","YARN"): - indexfile.write(" * [%s %s](%s/%s.%s.%s.html)\n" \ - % (p,k,v,k.upper().replace(" ",""),p.upper(),v)) indexfile.close() +class GetVersions: + """ yo """ + def __init__(self,versions, projects): + versions = versions + projects = projects + self.newversions = [] + pp = pprint.PrettyPrinter(indent=4) + at=0 + end=1 + count=100 + versions.sort() + print "Looking for %s through %s"%(versions[0],versions[-1]) + for p in projects: + resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p) + data = json.loads(resp.read()) + for d in data: + if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]: + print "Adding %s to the list" % d['name'] + self.newversions.append(d['name']) + newlist=list(set(self.newversions)) + self.newversions=newlist + + def getlist(self): + pp = pprint.PrettyPrinter(indent=4) + return(self.newversions) + class Version: """Represents a version number""" def __init__(self, data): @@ -261,8 +283,10 @@ class Jira: class JiraIter: """An Iterator of JIRAs""" - def __init__(self, versions): - self.versions = versions + def __init__(self, version, projects): + self.version = version + self.projects = projects + v=str(version).replace("-SNAPSHOT","") resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field") data = json.loads(resp.read()) @@ -276,7 +300,7 @@ class JiraIter: end=1 count=100 while (at < end): - params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count}) + params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count}) resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params) data = json.loads(resp.read()) if (data.has_key('errorMessages')): @@ -286,10 +310,8 @@ class JiraIter: self.jiras.extend(data['issues']) needaversion=False - for j in versions: - v=str(j).replace("-SNAPSHOT","") - if v not in releaseVersion: - needaversion=True + if v not in releaseVersion: + needaversion=True if needaversion is True: for i in range(len(data['issues'])): @@ -351,21 +373,29 @@ class Outputs: self.writeKeyRaw(jira.getProject(), line) def main(): - parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]", + parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]", epilog= "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory" " named after the highest version provided.") + parser.add_option("-i","--index", dest="index", action="store_true", + default=False, help="build an index file") + parser.add_option("-l","--license", dest="license", action="store_false", + default=True, help="Add an ASF license") + parser.add_option("-n","--lint", dest="lint", action="store_true", + help="use lint flag to exit on failures") + parser.add_option("-p", "--project", dest="projects", + action="append", type="string", + help="projects in JIRA to include in releasenotes", metavar="PROJECT") + parser.add_option("-r", "--range", dest="range", action="store_true", + default=False, help="Given versions are a range") + parser.add_option("-t", "--projecttitle", dest="title", + type="string", + help="Title to use for the project (default is Apache PROJECT)") + parser.add_option("-u","--usetoday", dest="usetoday", action="store_true", + default=False, help="use current date for unreleased versions") parser.add_option("-v", "--version", dest="versions", action="append", type="string", help="versions in JIRA to include in releasenotes", metavar="VERSION") - parser.add_option("-m","--master", dest="master", action="store_true", - help="only create the master, merged project files") - parser.add_option("-i","--index", dest="index", action="store_true", - help="build an index file") - parser.add_option("-u","--usetoday", dest="usetoday", action="store_true", - help="use current date for unreleased versions") - parser.add_option("-n","--lint", dest="lint", action="store_true", - help="use lint flag to exit on failures") (options, args) = parser.parse_args() if (options.versions is None): @@ -377,169 +407,169 @@ def main(): if (len(options.versions) <= 0): parser.error("At least one version needs to be supplied") - versions = [ Version(v) for v in options.versions ]; + projects = options.projects + + if (options.range is True): + versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ] + else: + versions = [ Version(v) for v in options.versions ] versions.sort(); - maxVersion = str(versions[-1]) - - jlist = JiraIter(versions) - version = maxVersion - - if version in releaseVersion: - reldate=releaseVersion[version] - elif options.usetoday: - reldate=strftime("%Y-%m-%d", gmtime()) + if (options.title is None): + title=projects[0] else: - reldate="Unreleased" + title=options.title - if not os.path.exists(version): - os.mkdir(version) + for v in versions: + vstr=str(v) + jlist = JiraIter(vstr,projects) - if options.master: - reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", - "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", - [], {"ver":maxVersion, "date":reldate}) - choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md", - "%(ver)s/CHANGES.%(key)s.%(ver)s.md", - [], {"ver":maxVersion, "date":reldate}) - else: - reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", - "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", - ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate}) - choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md", - "%(ver)s/CHANGES.%(key)s.%(ver)s.md", - ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate}) - - reloutputs.writeAll(asflicense) - choutputs.writeAll(asflicense) - - relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \ - 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n' - - chhead = '# Hadoop Changelog\n\n' \ - '## Release %(ver)s - %(date)s\n'\ - '\n' - - reloutputs.writeAll(relhead) - choutputs.writeAll(chhead) - - errorCount=0 - warningCount=0 - lintMessage="" - incompatlist=[] - buglist=[] - improvementlist=[] - newfeaturelist=[] - subtasklist=[] - tasklist=[] - testlist=[] - otherlist=[] - - for jira in sorted(jlist): - if jira.getIncompatibleChange(): - incompatlist.append(jira) - if (len(jira.getReleaseNote())==0): - warningCount+=1 - - if jira.checkVersionString(): - warningCount+=1 - - if jira.checkMissingComponent() or jira.checkMissingAssignee(): - errorCount+=1 - elif jira.getType() == "Bug": - buglist.append(jira) - elif jira.getType() == "Improvement": - improvementlist.append(jira) - elif jira.getType() == "New Feature": - newfeaturelist.append(jira) - elif jira.getType() == "Sub-task": - subtasklist.append(jira) - elif jira.getType() == "Task": - tasklist.append(jira) - elif jira.getType() == "Test": - testlist.append(jira) + if vstr in releaseVersion: + reldate=releaseVersion[vstr] + elif options.usetoday: + reldate=strftime("%Y-%m-%d", gmtime()) else: - otherlist.append(jira) + reldate="Unreleased" - line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \ - % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()), - notableclean(jira.getSummary())) + if not os.path.exists(vstr): + os.mkdir(vstr) - if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0): - reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") - reloutputs.writeKeyRaw(jira.getProject(), line) - line ='\n**WARNING: No release note provided for this incompatible change.**\n\n' - lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId())) - reloutputs.writeKeyRaw(jira.getProject(), line) + reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", + "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", + [], {"ver":v, "date":reldate, "title":title}) + choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md", + "%(ver)s/CHANGES.%(key)s.%(ver)s.md", + [], {"ver":v, "date":reldate, "title":title}) - if jira.checkVersionString(): - lintMessage += "\nWARNING: Version string problem for %s " % jira.getId() + if (options.license is True): + reloutputs.writeAll(asflicense) + choutputs.writeAll(asflicense) - if (jira.checkMissingComponent() or jira.checkMissingAssignee()): - errorMessage=[] - jira.checkMissingComponent() and errorMessage.append("component") - jira.checkMissingAssignee() and errorMessage.append("assignee") - lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId()) + relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \ + 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n' + chhead = '# %(title)s Changelog\n\n' \ + '## Release %(ver)s - %(date)s\n'\ + '\n' - if (len(jira.getReleaseNote())>0): - reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") - reloutputs.writeKeyRaw(jira.getProject(), line) - line ='\n%s\n\n' % (tableclean(jira.getReleaseNote())) - reloutputs.writeKeyRaw(jira.getProject(), line) + reloutputs.writeAll(relhead) + choutputs.writeAll(chhead) + errorCount=0 + warningCount=0 + lintMessage="" + incompatlist=[] + buglist=[] + improvementlist=[] + newfeaturelist=[] + subtasklist=[] + tasklist=[] + testlist=[] + otherlist=[] - if (options.lint is True): - print lintMessage - print "=======================================" - print "Error:%d, Warning:%d \n" % (errorCount, warningCount) + for jira in sorted(jlist): + if jira.getIncompatibleChange(): + incompatlist.append(jira) + if (len(jira.getReleaseNote())==0): + warningCount+=1 - if (errorCount>0): - cleanOutputDir(version) - sys.exit(1) + if jira.checkVersionString(): + warningCount+=1 - reloutputs.writeAll("\n\n") - reloutputs.close() + if jira.checkMissingComponent() or jira.checkMissingAssignee(): + errorCount+=1 + elif jira.getType() == "Bug": + buglist.append(jira) + elif jira.getType() == "Improvement": + improvementlist.append(jira) + elif jira.getType() == "New Feature": + newfeaturelist.append(jira) + elif jira.getType() == "Sub-task": + subtasklist.append(jira) + elif jira.getType() == "Task": + tasklist.append(jira) + elif jira.getType() == "Test": + testlist.append(jira) + else: + otherlist.append(jira) - choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(incompatlist) + line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \ + % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()), + notableclean(jira.getSummary())) - choutputs.writeAll("\n\n### NEW FEATURES:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(newfeaturelist) + if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0): + reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") + reloutputs.writeKeyRaw(jira.getProject(), line) + line ='\n**WARNING: No release note provided for this incompatible change.**\n\n' + lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId())) + reloutputs.writeKeyRaw(jira.getProject(), line) - choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(improvementlist) + if jira.checkVersionString(): + lintMessage += "\nWARNING: Version string problem for %s " % jira.getId() - choutputs.writeAll("\n\n### BUG FIXES:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(buglist) + if (jira.checkMissingComponent() or jira.checkMissingAssignee()): + errorMessage=[] + jira.checkMissingComponent() and errorMessage.append("component") + jira.checkMissingAssignee() and errorMessage.append("assignee") + lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId()) - choutputs.writeAll("\n\n### TESTS:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(testlist) + if (len(jira.getReleaseNote())>0): + reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") + reloutputs.writeKeyRaw(jira.getProject(), line) + line ='\n%s\n\n' % (tableclean(jira.getReleaseNote())) + reloutputs.writeKeyRaw(jira.getProject(), line) - choutputs.writeAll("\n\n### SUB-TASKS:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(subtasklist) + if (options.lint is True): + print lintMessage + print "=======================================" + print "Error:%d, Warning:%d \n" % (errorCount, warningCount) - choutputs.writeAll("\n\n### OTHER:\n\n") - choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.writeList(otherlist) - choutputs.writeList(tasklist) + if (errorCount>0): + cleanOutputDir(version) + sys.exit(1) - choutputs.writeAll("\n\n") - choutputs.close() + reloutputs.writeAll("\n\n") + reloutputs.close() + + choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(incompatlist) + + choutputs.writeAll("\n\n### NEW FEATURES:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(newfeaturelist) + + choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(improvementlist) + + choutputs.writeAll("\n\n### BUG FIXES:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(buglist) + + choutputs.writeAll("\n\n### TESTS:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(testlist) + + choutputs.writeAll("\n\n### SUB-TASKS:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(subtasklist) + + choutputs.writeAll("\n\n### OTHER:\n\n") + choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") + choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") + choutputs.writeList(otherlist) + choutputs.writeList(tasklist) + + choutputs.writeAll("\n\n") + choutputs.close() if options.index: - buildindex(options.master) + buildindex(title,options.license) if __name__ == "__main__": main()