HADOOP-12135. cleanup releasedocmaker

(cherry picked from commit 3fee9f8d18)
(cherry picked from commit e8b62d11d4)
This commit is contained in:
Allen Wittenauer 2015-07-06 15:49:03 -07:00 committed by Andrew Wang
parent cf348f2eb8
commit a6cad7d856
1 changed files with 195 additions and 165 deletions

View File

@ -19,6 +19,7 @@
from glob import glob from glob import glob
from optparse import OptionParser from optparse import OptionParser
from time import gmtime, strftime from time import gmtime, strftime
import pprint
import os import os
import re import re
import sys import sys
@ -99,23 +100,44 @@ def mstr(obj):
return "" return ""
return unicode(obj) return unicode(obj)
def buildindex(master): def buildindex(title,license):
versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*"))) versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
with open("index.md","w") as indexfile: with open("index.md","w") as indexfile:
if license is True:
indexfile.write(asflicense)
for v in versions: for v in versions:
indexfile.write("* Apache Hadoop v%s\n" % (v)) indexfile.write("* %s v%s\n" % (title,v))
for k in ("Changes","Release Notes"): for k in ("Changes","Release Notes"):
indexfile.write(" * %s\n" %(k)) indexfile.write(" * %s (%s/%s.%s.html)\n" \
indexfile.write(" * [Combined %s](%s/%s.%s.html)\n" \
% (k,v,k.upper().replace(" ",""),v)) % (k,v,k.upper().replace(" ",""),v))
if not master:
indexfile.write(" * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \
% (k,v,k.upper().replace(" ",""),v))
for p in ("HDFS","MapReduce","YARN"):
indexfile.write(" * [%s %s](%s/%s.%s.%s.html)\n" \
% (p,k,v,k.upper().replace(" ",""),p.upper(),v))
indexfile.close() indexfile.close()
class GetVersions:
""" yo """
def __init__(self,versions, projects):
versions = versions
projects = projects
self.newversions = []
pp = pprint.PrettyPrinter(indent=4)
at=0
end=1
count=100
versions.sort()
print "Looking for %s through %s"%(versions[0],versions[-1])
for p in projects:
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
data = json.loads(resp.read())
for d in data:
if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
print "Adding %s to the list" % d['name']
self.newversions.append(d['name'])
newlist=list(set(self.newversions))
self.newversions=newlist
def getlist(self):
pp = pprint.PrettyPrinter(indent=4)
return(self.newversions)
class Version: class Version:
"""Represents a version number""" """Represents a version number"""
def __init__(self, data): def __init__(self, data):
@ -261,8 +283,10 @@ class Jira:
class JiraIter: class JiraIter:
"""An Iterator of JIRAs""" """An Iterator of JIRAs"""
def __init__(self, versions): def __init__(self, version, projects):
self.versions = versions self.version = version
self.projects = projects
v=str(version).replace("-SNAPSHOT","")
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field") resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
data = json.loads(resp.read()) data = json.loads(resp.read())
@ -276,7 +300,7 @@ class JiraIter:
end=1 end=1
count=100 count=100
while (at < end): while (at < end):
params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count}) params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params) resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
data = json.loads(resp.read()) data = json.loads(resp.read())
if (data.has_key('errorMessages')): if (data.has_key('errorMessages')):
@ -286,10 +310,8 @@ class JiraIter:
self.jiras.extend(data['issues']) self.jiras.extend(data['issues'])
needaversion=False needaversion=False
for j in versions: if v not in releaseVersion:
v=str(j).replace("-SNAPSHOT","") needaversion=True
if v not in releaseVersion:
needaversion=True
if needaversion is True: if needaversion is True:
for i in range(len(data['issues'])): for i in range(len(data['issues'])):
@ -351,21 +373,29 @@ class Outputs:
self.writeKeyRaw(jira.getProject(), line) self.writeKeyRaw(jira.getProject(), line)
def main(): def main():
parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]", parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
epilog= epilog=
"Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory" "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
" named after the highest version provided.") " named after the highest version provided.")
parser.add_option("-i","--index", dest="index", action="store_true",
default=False, help="build an index file")
parser.add_option("-l","--license", dest="license", action="store_false",
default=True, help="Add an ASF license")
parser.add_option("-n","--lint", dest="lint", action="store_true",
help="use lint flag to exit on failures")
parser.add_option("-p", "--project", dest="projects",
action="append", type="string",
help="projects in JIRA to include in releasenotes", metavar="PROJECT")
parser.add_option("-r", "--range", dest="range", action="store_true",
default=False, help="Given versions are a range")
parser.add_option("-t", "--projecttitle", dest="title",
type="string",
help="Title to use for the project (default is Apache PROJECT)")
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
default=False, help="use current date for unreleased versions")
parser.add_option("-v", "--version", dest="versions", parser.add_option("-v", "--version", dest="versions",
action="append", type="string", action="append", type="string",
help="versions in JIRA to include in releasenotes", metavar="VERSION") help="versions in JIRA to include in releasenotes", metavar="VERSION")
parser.add_option("-m","--master", dest="master", action="store_true",
help="only create the master, merged project files")
parser.add_option("-i","--index", dest="index", action="store_true",
help="build an index file")
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
help="use current date for unreleased versions")
parser.add_option("-n","--lint", dest="lint", action="store_true",
help="use lint flag to exit on failures")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if (options.versions is None): if (options.versions is None):
@ -377,169 +407,169 @@ def main():
if (len(options.versions) <= 0): if (len(options.versions) <= 0):
parser.error("At least one version needs to be supplied") parser.error("At least one version needs to be supplied")
versions = [ Version(v) for v in options.versions ]; projects = options.projects
if (options.range is True):
versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
else:
versions = [ Version(v) for v in options.versions ]
versions.sort(); versions.sort();
maxVersion = str(versions[-1]) if (options.title is None):
title=projects[0]
jlist = JiraIter(versions)
version = maxVersion
if version in releaseVersion:
reldate=releaseVersion[version]
elif options.usetoday:
reldate=strftime("%Y-%m-%d", gmtime())
else: else:
reldate="Unreleased" title=options.title
if not os.path.exists(version): for v in versions:
os.mkdir(version) vstr=str(v)
jlist = JiraIter(vstr,projects)
if options.master: if vstr in releaseVersion:
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", reldate=releaseVersion[vstr]
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", elif options.usetoday:
[], {"ver":maxVersion, "date":reldate}) reldate=strftime("%Y-%m-%d", gmtime())
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
[], {"ver":maxVersion, "date":reldate})
else:
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
reloutputs.writeAll(asflicense)
choutputs.writeAll(asflicense)
relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
chhead = '# Hadoop Changelog\n\n' \
'## Release %(ver)s - %(date)s\n'\
'\n'
reloutputs.writeAll(relhead)
choutputs.writeAll(chhead)
errorCount=0
warningCount=0
lintMessage=""
incompatlist=[]
buglist=[]
improvementlist=[]
newfeaturelist=[]
subtasklist=[]
tasklist=[]
testlist=[]
otherlist=[]
for jira in sorted(jlist):
if jira.getIncompatibleChange():
incompatlist.append(jira)
if (len(jira.getReleaseNote())==0):
warningCount+=1
if jira.checkVersionString():
warningCount+=1
if jira.checkMissingComponent() or jira.checkMissingAssignee():
errorCount+=1
elif jira.getType() == "Bug":
buglist.append(jira)
elif jira.getType() == "Improvement":
improvementlist.append(jira)
elif jira.getType() == "New Feature":
newfeaturelist.append(jira)
elif jira.getType() == "Sub-task":
subtasklist.append(jira)
elif jira.getType() == "Task":
tasklist.append(jira)
elif jira.getType() == "Test":
testlist.append(jira)
else: else:
otherlist.append(jira) reldate="Unreleased"
line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \ if not os.path.exists(vstr):
% (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()), os.mkdir(vstr)
notableclean(jira.getSummary()))
if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0): reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
reloutputs.writeKeyRaw(jira.getProject(), line) [], {"ver":v, "date":reldate, "title":title})
line ='\n**WARNING: No release note provided for this incompatible change.**\n\n' choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId())) "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
reloutputs.writeKeyRaw(jira.getProject(), line) [], {"ver":v, "date":reldate, "title":title})
if jira.checkVersionString(): if (options.license is True):
lintMessage += "\nWARNING: Version string problem for %s " % jira.getId() reloutputs.writeAll(asflicense)
choutputs.writeAll(asflicense)
if (jira.checkMissingComponent() or jira.checkMissingAssignee()): relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
errorMessage=[] 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
jira.checkMissingComponent() and errorMessage.append("component") chhead = '# %(title)s Changelog\n\n' \
jira.checkMissingAssignee() and errorMessage.append("assignee") '## Release %(ver)s - %(date)s\n'\
lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId()) '\n'
if (len(jira.getReleaseNote())>0): reloutputs.writeAll(relhead)
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") choutputs.writeAll(chhead)
reloutputs.writeKeyRaw(jira.getProject(), line) errorCount=0
line ='\n%s\n\n' % (tableclean(jira.getReleaseNote())) warningCount=0
reloutputs.writeKeyRaw(jira.getProject(), line) lintMessage=""
incompatlist=[]
buglist=[]
improvementlist=[]
newfeaturelist=[]
subtasklist=[]
tasklist=[]
testlist=[]
otherlist=[]
if (options.lint is True): for jira in sorted(jlist):
print lintMessage if jira.getIncompatibleChange():
print "=======================================" incompatlist.append(jira)
print "Error:%d, Warning:%d \n" % (errorCount, warningCount) if (len(jira.getReleaseNote())==0):
warningCount+=1
if (errorCount>0): if jira.checkVersionString():
cleanOutputDir(version) warningCount+=1
sys.exit(1)
reloutputs.writeAll("\n\n") if jira.checkMissingComponent() or jira.checkMissingAssignee():
reloutputs.close() errorCount+=1
elif jira.getType() == "Bug":
buglist.append(jira)
elif jira.getType() == "Improvement":
improvementlist.append(jira)
elif jira.getType() == "New Feature":
newfeaturelist.append(jira)
elif jira.getType() == "Sub-task":
subtasklist.append(jira)
elif jira.getType() == "Task":
tasklist.append(jira)
elif jira.getType() == "Test":
testlist.append(jira)
else:
otherlist.append(jira)
choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n") line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") notableclean(jira.getSummary()))
choutputs.writeList(incompatlist)
choutputs.writeAll("\n\n### NEW FEATURES:\n\n") if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") reloutputs.writeKeyRaw(jira.getProject(), line)
choutputs.writeList(newfeaturelist) line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
reloutputs.writeKeyRaw(jira.getProject(), line)
choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n") if jira.checkVersionString():
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(improvementlist)
choutputs.writeAll("\n\n### BUG FIXES:\n\n") if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") errorMessage=[]
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") jira.checkMissingComponent() and errorMessage.append("component")
choutputs.writeList(buglist) jira.checkMissingAssignee() and errorMessage.append("assignee")
lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
choutputs.writeAll("\n\n### TESTS:\n\n") if (len(jira.getReleaseNote())>0):
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") reloutputs.writeKeyRaw(jira.getProject(), line)
choutputs.writeList(testlist) line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
reloutputs.writeKeyRaw(jira.getProject(), line)
choutputs.writeAll("\n\n### SUB-TASKS:\n\n") if (options.lint is True):
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") print lintMessage
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") print "======================================="
choutputs.writeList(subtasklist) print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
choutputs.writeAll("\n\n### OTHER:\n\n") if (errorCount>0):
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") cleanOutputDir(version)
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") sys.exit(1)
choutputs.writeList(otherlist)
choutputs.writeList(tasklist)
choutputs.writeAll("\n\n") reloutputs.writeAll("\n\n")
choutputs.close() reloutputs.close()
choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(incompatlist)
choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(newfeaturelist)
choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(improvementlist)
choutputs.writeAll("\n\n### BUG FIXES:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(buglist)
choutputs.writeAll("\n\n### TESTS:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(testlist)
choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(subtasklist)
choutputs.writeAll("\n\n### OTHER:\n\n")
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
choutputs.writeList(otherlist)
choutputs.writeList(tasklist)
choutputs.writeAll("\n\n")
choutputs.close()
if options.index: if options.index:
buildindex(options.master) buildindex(title,options.license)
if __name__ == "__main__": if __name__ == "__main__":
main() main()