HADOOP-12651. Replace dev-support with wrappers to Yetus (aw)
This commit is contained in:
Allen Wittenauer
parent
c304890c8c
commit
8cecad2d56
|
|
@ -39,7 +39,7 @@ and all required tools for testing and building have been installed and configur
|
|||
|
||||
Note that from within this docker environment you ONLY have access to the Hadoop source
|
||||
tree from where you started. So if you need to run
|
||||
dev-support/test-patch.sh /path/to/my.patch
|
||||
dev-support/bin/test-patch /path/to/my.patch
|
||||
then the patch must be placed inside the hadoop source tree.
|
||||
|
||||
Known issues:
|
||||
|
|
|
|||
|
|
@ -1,580 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from glob import glob
|
||||
from optparse import OptionParser
|
||||
from time import gmtime, strftime
|
||||
import pprint
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib
|
||||
import urllib2
|
||||
try:
|
||||
import json
|
||||
except ImportError:
|
||||
import simplejson as json
|
||||
|
||||
releaseVersion={}
|
||||
namePattern = re.compile(r' \([0-9]+\)')
|
||||
|
||||
asflicense='''
|
||||
<!---
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
-->
|
||||
'''
|
||||
|
||||
def clean(str):
|
||||
return tableclean(re.sub(namePattern, "", str))
|
||||
|
||||
def formatComponents(str):
|
||||
str = re.sub(namePattern, '', str).replace("'", "")
|
||||
if str != "":
|
||||
ret = str
|
||||
else:
|
||||
# some markdown parsers don't like empty tables
|
||||
ret = "."
|
||||
return clean(ret)
|
||||
|
||||
# convert to utf-8
|
||||
# protect some known md metachars
|
||||
# or chars that screw up doxia
|
||||
def tableclean(str):
|
||||
str=str.encode('utf-8')
|
||||
str=str.replace("_","\_")
|
||||
str=str.replace("\r","")
|
||||
str=str.rstrip()
|
||||
return str
|
||||
|
||||
# same thing as tableclean,
|
||||
# except table metachars are also
|
||||
# escaped as well as more
|
||||
# things we don't want doxia to
|
||||
# screw up
|
||||
def notableclean(str):
|
||||
str=tableclean(str)
|
||||
str=str.replace("|","\|")
|
||||
str=str.replace("<","\<")
|
||||
str=str.replace(">","\>")
|
||||
str=str.replace("*","\*")
|
||||
str=str.rstrip()
|
||||
return str
|
||||
|
||||
# clean output dir
|
||||
def cleanOutputDir(dir):
|
||||
files = os.listdir(dir)
|
||||
for name in files:
|
||||
os.remove(os.path.join(dir,name))
|
||||
os.rmdir(dir)
|
||||
|
||||
def mstr(obj):
|
||||
if (obj is None):
|
||||
return ""
|
||||
return unicode(obj)
|
||||
|
||||
def buildindex(title,license):
|
||||
versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
|
||||
with open("index.md","w") as indexfile:
|
||||
if license is True:
|
||||
indexfile.write(asflicense)
|
||||
for v in versions:
|
||||
indexfile.write("* %s v%s\n" % (title,v))
|
||||
for k in ("Changes","Release Notes"):
|
||||
indexfile.write(" * %s (%s/%s.%s.html)\n" \
|
||||
% (k,v,k.upper().replace(" ",""),v))
|
||||
indexfile.close()
|
||||
|
||||
class GetVersions:
|
||||
""" yo """
|
||||
def __init__(self,versions, projects):
|
||||
versions = versions
|
||||
projects = projects
|
||||
self.newversions = []
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
at=0
|
||||
end=1
|
||||
count=100
|
||||
versions.sort()
|
||||
print "Looking for %s through %s"%(versions[0],versions[-1])
|
||||
for p in projects:
|
||||
resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
|
||||
data = json.loads(resp.read())
|
||||
for d in data:
|
||||
if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
|
||||
print "Adding %s to the list" % d['name']
|
||||
self.newversions.append(d['name'])
|
||||
newlist=list(set(self.newversions))
|
||||
self.newversions=newlist
|
||||
|
||||
def getlist(self):
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
return(self.newversions)
|
||||
|
||||
class Version:
|
||||
"""Represents a version number"""
|
||||
def __init__(self, data):
|
||||
self.mod = False
|
||||
self.data = data
|
||||
found = re.match('^((\d+)(\.\d+)*).*$', data)
|
||||
if (found):
|
||||
self.parts = [ int(p) for p in found.group(1).split('.') ]
|
||||
else:
|
||||
self.parts = []
|
||||
# backfill version with zeroes if missing parts
|
||||
self.parts.extend((0,) * (3 - len(self.parts)))
|
||||
|
||||
def __str__(self):
|
||||
if (self.mod):
|
||||
return '.'.join([ str(p) for p in self.parts ])
|
||||
return self.data
|
||||
|
||||
def __cmp__(self, other):
|
||||
return cmp(self.parts, other.parts)
|
||||
|
||||
class Jira:
|
||||
"""A single JIRA"""
|
||||
|
||||
def __init__(self, data, parent):
|
||||
self.key = data['key']
|
||||
self.fields = data['fields']
|
||||
self.parent = parent
|
||||
self.notes = None
|
||||
self.incompat = None
|
||||
self.reviewed = None
|
||||
|
||||
def getId(self):
|
||||
return mstr(self.key)
|
||||
|
||||
def getDescription(self):
|
||||
return mstr(self.fields['description'])
|
||||
|
||||
def getReleaseNote(self):
|
||||
if (self.notes is None):
|
||||
field = self.parent.fieldIdMap['Release Note']
|
||||
if (self.fields.has_key(field)):
|
||||
self.notes=mstr(self.fields[field])
|
||||
else:
|
||||
self.notes=self.getDescription()
|
||||
return self.notes
|
||||
|
||||
def getPriority(self):
|
||||
ret = ""
|
||||
pri = self.fields['priority']
|
||||
if(pri is not None):
|
||||
ret = pri['name']
|
||||
return mstr(ret)
|
||||
|
||||
def getAssignee(self):
|
||||
ret = ""
|
||||
mid = self.fields['assignee']
|
||||
if(mid is not None):
|
||||
ret = mid['displayName']
|
||||
return mstr(ret)
|
||||
|
||||
def getComponents(self):
|
||||
if (len(self.fields['components'])>0):
|
||||
return ", ".join([ comp['name'] for comp in self.fields['components'] ])
|
||||
else:
|
||||
return ""
|
||||
|
||||
def getSummary(self):
|
||||
return self.fields['summary']
|
||||
|
||||
def getType(self):
|
||||
ret = ""
|
||||
mid = self.fields['issuetype']
|
||||
if(mid is not None):
|
||||
ret = mid['name']
|
||||
return mstr(ret)
|
||||
|
||||
def getReporter(self):
|
||||
ret = ""
|
||||
mid = self.fields['reporter']
|
||||
if(mid is not None):
|
||||
ret = mid['displayName']
|
||||
return mstr(ret)
|
||||
|
||||
def getProject(self):
|
||||
ret = ""
|
||||
mid = self.fields['project']
|
||||
if(mid is not None):
|
||||
ret = mid['key']
|
||||
return mstr(ret)
|
||||
|
||||
def __cmp__(self,other):
|
||||
selfsplit=self.getId().split('-')
|
||||
othersplit=other.getId().split('-')
|
||||
v1=cmp(selfsplit[0],othersplit[0])
|
||||
if (v1!=0):
|
||||
return v1
|
||||
else:
|
||||
if selfsplit[1] < othersplit[1]:
|
||||
return True
|
||||
elif selfsplit[1] > othersplit[1]:
|
||||
return False
|
||||
return False
|
||||
|
||||
def getIncompatibleChange(self):
|
||||
if (self.incompat is None):
|
||||
field = self.parent.fieldIdMap['Hadoop Flags']
|
||||
self.reviewed=False
|
||||
self.incompat=False
|
||||
if (self.fields.has_key(field)):
|
||||
if self.fields[field]:
|
||||
for hf in self.fields[field]:
|
||||
if hf['value'] == "Incompatible change":
|
||||
self.incompat=True
|
||||
if hf['value'] == "Reviewed":
|
||||
self.reviewed=True
|
||||
return self.incompat
|
||||
|
||||
def checkMissingComponent(self):
|
||||
if (len(self.fields['components'])>0):
|
||||
return False
|
||||
return True
|
||||
|
||||
def checkMissingAssignee(self):
|
||||
if (self.fields['assignee'] is not None):
|
||||
return False
|
||||
return True
|
||||
|
||||
def checkVersionString(self):
|
||||
field = self.parent.fieldIdMap['Fix Version/s']
|
||||
for h in self.fields[field]:
|
||||
found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
|
||||
if not found:
|
||||
return True
|
||||
return False
|
||||
|
||||
def getReleaseDate(self,version):
|
||||
for j in range(len(self.fields['fixVersions'])):
|
||||
if self.fields['fixVersions'][j]==version:
|
||||
return(self.fields['fixVersions'][j]['releaseDate'])
|
||||
return None
|
||||
|
||||
class JiraIter:
|
||||
"""An Iterator of JIRAs"""
|
||||
|
||||
def __init__(self, version, projects):
|
||||
self.version = version
|
||||
self.projects = projects
|
||||
v=str(version).replace("-SNAPSHOT","")
|
||||
|
||||
resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field")
|
||||
data = json.loads(resp.read())
|
||||
|
||||
self.fieldIdMap = {}
|
||||
for part in data:
|
||||
self.fieldIdMap[part['name']] = part['id']
|
||||
|
||||
self.jiras = []
|
||||
at=0
|
||||
end=1
|
||||
count=100
|
||||
while (at < end):
|
||||
params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
|
||||
resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
|
||||
data = json.loads(resp.read())
|
||||
if (data.has_key('errorMessages')):
|
||||
raise Exception(data['errorMessages'])
|
||||
at = data['startAt'] + data['maxResults']
|
||||
end = data['total']
|
||||
self.jiras.extend(data['issues'])
|
||||
|
||||
needaversion=False
|
||||
if v not in releaseVersion:
|
||||
needaversion=True
|
||||
|
||||
if needaversion is True:
|
||||
for i in range(len(data['issues'])):
|
||||
for j in range(len(data['issues'][i]['fields']['fixVersions'])):
|
||||
if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
|
||||
releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
|
||||
data['issues'][i]['fields']['fixVersions'][j]['releaseDate']
|
||||
|
||||
self.iter = self.jiras.__iter__()
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
data = self.iter.next()
|
||||
j = Jira(data, self)
|
||||
return j
|
||||
|
||||
class Outputs:
|
||||
"""Several different files to output to at the same time"""
|
||||
|
||||
def __init__(self, base_file_name, file_name_pattern, keys, params={}):
|
||||
self.params = params
|
||||
self.base = open(base_file_name%params, 'w')
|
||||
self.others = {}
|
||||
for key in keys:
|
||||
both = dict(params)
|
||||
both['key'] = key
|
||||
self.others[key] = open(file_name_pattern%both, 'w')
|
||||
|
||||
def writeAll(self, pattern):
|
||||
both = dict(self.params)
|
||||
both['key'] = ''
|
||||
self.base.write(pattern%both)
|
||||
for key in self.others.keys():
|
||||
both = dict(self.params)
|
||||
both['key'] = key
|
||||
self.others[key].write(pattern%both)
|
||||
|
||||
def writeKeyRaw(self, key, str):
|
||||
self.base.write(str)
|
||||
if (self.others.has_key(key)):
|
||||
self.others[key].write(str)
|
||||
|
||||
def close(self):
|
||||
self.base.close()
|
||||
for fd in self.others.values():
|
||||
fd.close()
|
||||
|
||||
def writeList(self, mylist):
|
||||
for jira in sorted(mylist):
|
||||
line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' \
|
||||
% (notableclean(jira.getId()), notableclean(jira.getId()),
|
||||
notableclean(jira.getSummary()),
|
||||
notableclean(jira.getPriority()),
|
||||
formatComponents(jira.getComponents()),
|
||||
notableclean(jira.getReporter()),
|
||||
notableclean(jira.getAssignee()))
|
||||
self.writeKeyRaw(jira.getProject(), line)
|
||||
|
||||
def main():
|
||||
parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
|
||||
epilog=
|
||||
"Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
|
||||
" named after the highest version provided.")
|
||||
parser.add_option("-i","--index", dest="index", action="store_true",
|
||||
default=False, help="build an index file")
|
||||
parser.add_option("-l","--license", dest="license", action="store_false",
|
||||
default=True, help="Add an ASF license")
|
||||
parser.add_option("-n","--lint", dest="lint", action="store_true",
|
||||
help="use lint flag to exit on failures")
|
||||
parser.add_option("-p", "--project", dest="projects",
|
||||
action="append", type="string",
|
||||
help="projects in JIRA to include in releasenotes", metavar="PROJECT")
|
||||
parser.add_option("-r", "--range", dest="range", action="store_true",
|
||||
default=False, help="Given versions are a range")
|
||||
parser.add_option("-t", "--projecttitle", dest="title",
|
||||
type="string",
|
||||
help="Title to use for the project (default is Apache PROJECT)")
|
||||
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
|
||||
default=False, help="use current date for unreleased versions")
|
||||
parser.add_option("-v", "--version", dest="versions",
|
||||
action="append", type="string",
|
||||
help="versions in JIRA to include in releasenotes", metavar="VERSION")
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if (options.versions is None):
|
||||
options.versions = []
|
||||
|
||||
if (len(args) > 2):
|
||||
options.versions.append(args[2])
|
||||
|
||||
if (len(options.versions) <= 0):
|
||||
parser.error("At least one version needs to be supplied")
|
||||
|
||||
proxy = urllib2.ProxyHandler()
|
||||
opener = urllib2.build_opener(proxy)
|
||||
urllib2.install_opener(opener)
|
||||
|
||||
projects = options.projects
|
||||
|
||||
if (options.range is True):
|
||||
versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
|
||||
else:
|
||||
versions = [ Version(v) for v in options.versions ]
|
||||
versions.sort();
|
||||
|
||||
if (options.title is None):
|
||||
title=projects[0]
|
||||
else:
|
||||
title=options.title
|
||||
|
||||
haderrors=False
|
||||
|
||||
for v in versions:
|
||||
vstr=str(v)
|
||||
jlist = JiraIter(vstr,projects)
|
||||
|
||||
if vstr in releaseVersion:
|
||||
reldate=releaseVersion[vstr]
|
||||
elif options.usetoday:
|
||||
reldate=strftime("%Y-%m-%d", gmtime())
|
||||
else:
|
||||
reldate="Unreleased"
|
||||
|
||||
if not os.path.exists(vstr):
|
||||
os.mkdir(vstr)
|
||||
|
||||
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
|
||||
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
|
||||
[], {"ver":v, "date":reldate, "title":title})
|
||||
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
|
||||
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
|
||||
[], {"ver":v, "date":reldate, "title":title})
|
||||
|
||||
if (options.license is True):
|
||||
reloutputs.writeAll(asflicense)
|
||||
choutputs.writeAll(asflicense)
|
||||
|
||||
relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
|
||||
'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
|
||||
chhead = '# %(title)s Changelog\n\n' \
|
||||
'## Release %(ver)s - %(date)s\n'\
|
||||
'\n'
|
||||
|
||||
reloutputs.writeAll(relhead)
|
||||
choutputs.writeAll(chhead)
|
||||
errorCount=0
|
||||
warningCount=0
|
||||
lintMessage=""
|
||||
incompatlist=[]
|
||||
buglist=[]
|
||||
improvementlist=[]
|
||||
newfeaturelist=[]
|
||||
subtasklist=[]
|
||||
tasklist=[]
|
||||
testlist=[]
|
||||
otherlist=[]
|
||||
|
||||
for jira in sorted(jlist):
|
||||
if jira.getIncompatibleChange():
|
||||
incompatlist.append(jira)
|
||||
elif jira.getType() == "Bug":
|
||||
buglist.append(jira)
|
||||
elif jira.getType() == "Improvement":
|
||||
improvementlist.append(jira)
|
||||
elif jira.getType() == "New Feature":
|
||||
newfeaturelist.append(jira)
|
||||
elif jira.getType() == "Sub-task":
|
||||
subtasklist.append(jira)
|
||||
elif jira.getType() == "Task":
|
||||
tasklist.append(jira)
|
||||
elif jira.getType() == "Test":
|
||||
testlist.append(jira)
|
||||
else:
|
||||
otherlist.append(jira)
|
||||
|
||||
line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
|
||||
% (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
|
||||
notableclean(jira.getSummary()))
|
||||
|
||||
if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
|
||||
warningCount+=1
|
||||
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
|
||||
lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
|
||||
if jira.checkVersionString():
|
||||
warningCount+=1
|
||||
lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
|
||||
|
||||
if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
|
||||
errorCount+=1
|
||||
errorMessage=[]
|
||||
jira.checkMissingComponent() and errorMessage.append("component")
|
||||
jira.checkMissingAssignee() and errorMessage.append("assignee")
|
||||
lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
|
||||
|
||||
if (len(jira.getReleaseNote())>0):
|
||||
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
|
||||
if (options.lint is True):
|
||||
print lintMessage
|
||||
print "======================================="
|
||||
print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
|
||||
if (errorCount>0):
|
||||
haderrors=True
|
||||
cleanOutputDir(vstr)
|
||||
continue
|
||||
|
||||
reloutputs.writeAll("\n\n")
|
||||
reloutputs.close()
|
||||
|
||||
choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(incompatlist)
|
||||
|
||||
choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(newfeaturelist)
|
||||
|
||||
choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(improvementlist)
|
||||
|
||||
choutputs.writeAll("\n\n### BUG FIXES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(buglist)
|
||||
|
||||
choutputs.writeAll("\n\n### TESTS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(testlist)
|
||||
|
||||
choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(subtasklist)
|
||||
|
||||
choutputs.writeAll("\n\n### OTHER:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(otherlist)
|
||||
choutputs.writeList(tasklist)
|
||||
|
||||
choutputs.writeAll("\n\n")
|
||||
choutputs.close()
|
||||
|
||||
if options.index:
|
||||
buildindex(title,options.license)
|
||||
|
||||
if haderrors is True:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,271 +0,0 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
import sys
|
||||
import string
|
||||
from optparse import OptionParser
|
||||
|
||||
asflicense='''
|
||||
<!---
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
-->
|
||||
'''
|
||||
|
||||
def docstrip(key,string):
|
||||
string=re.sub("^## @%s " % key ,"",string)
|
||||
string=string.lstrip()
|
||||
string=string.rstrip()
|
||||
return string
|
||||
|
||||
def toc(list):
|
||||
tocout=[]
|
||||
header=()
|
||||
for i in list:
|
||||
if header != i.getinter():
|
||||
header=i.getinter()
|
||||
line=" * %s\n" % (i.headerbuild())
|
||||
tocout.append(line)
|
||||
line=" * [%s](#%s)\n" % (i.getname().replace("_","\_"),i.getname())
|
||||
tocout.append(line)
|
||||
return tocout
|
||||
|
||||
class ShellFunction:
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def __cmp__(self,other):
|
||||
if (self.audience == other.audience):
|
||||
if (self.stability == other.stability):
|
||||
if (self.replaceb == other.replaceb):
|
||||
return(cmp(self.name,other.name))
|
||||
else:
|
||||
if (self.replaceb == "Yes"):
|
||||
return -1
|
||||
else:
|
||||
return 1
|
||||
else:
|
||||
if (self.stability == "Stable"):
|
||||
return -1
|
||||
else:
|
||||
return 1
|
||||
else:
|
||||
if (self.audience == "Public"):
|
||||
return -1
|
||||
else:
|
||||
return 1
|
||||
|
||||
def reset(self):
|
||||
self.name=None
|
||||
self.audience=None
|
||||
self.stability=None
|
||||
self.replaceb=None
|
||||
self.returnt=None
|
||||
self.desc=None
|
||||
self.params=None
|
||||
|
||||
def setname(self,text):
|
||||
definition=text.split();
|
||||
self.name=definition[1]
|
||||
|
||||
def getname(self):
|
||||
if (self.name is None):
|
||||
return "None"
|
||||
else:
|
||||
return self.name
|
||||
|
||||
def setaudience(self,text):
|
||||
self.audience=docstrip("audience",text)
|
||||
self.audience=self.audience.capitalize()
|
||||
|
||||
def getaudience(self):
|
||||
if (self.audience is None):
|
||||
return "None"
|
||||
else:
|
||||
return self.audience
|
||||
|
||||
def setstability(self,text):
|
||||
self.stability=docstrip("stability",text)
|
||||
self.stability=self.stability.capitalize()
|
||||
|
||||
def getstability(self):
|
||||
if (self.stability is None):
|
||||
return "None"
|
||||
else:
|
||||
return self.stability
|
||||
|
||||
def setreplace(self,text):
|
||||
self.replaceb=docstrip("replaceable",text)
|
||||
self.replaceb=self.replaceb.capitalize()
|
||||
|
||||
def getreplace(self):
|
||||
if (self.replaceb is None):
|
||||
return "None"
|
||||
else:
|
||||
return self.replaceb
|
||||
|
||||
def getinter(self):
|
||||
return( (self.getaudience(), self.getstability(), self.getreplace()))
|
||||
|
||||
def addreturn(self,text):
|
||||
if (self.returnt is None):
|
||||
self.returnt = []
|
||||
self.returnt.append(docstrip("return",text))
|
||||
|
||||
def getreturn(self):
|
||||
if (self.returnt is None):
|
||||
return "Nothing"
|
||||
else:
|
||||
return "\n\n".join(self.returnt)
|
||||
|
||||
def adddesc(self,text):
|
||||
if (self.desc is None):
|
||||
self.desc = []
|
||||
self.desc.append(docstrip("description",text))
|
||||
|
||||
def getdesc(self):
|
||||
if (self.desc is None):
|
||||
return "None"
|
||||
else:
|
||||
return " ".join(self.desc)
|
||||
|
||||
def addparam(self,text):
|
||||
if (self.params is None):
|
||||
self.params = []
|
||||
self.params.append(docstrip("param",text))
|
||||
|
||||
def getparams(self):
|
||||
if (self.params is None):
|
||||
return ""
|
||||
else:
|
||||
return " ".join(self.params)
|
||||
|
||||
def getusage(self):
|
||||
line="%s %s" % (self.name, self.getparams())
|
||||
return line
|
||||
|
||||
def headerbuild(self):
|
||||
if self.getreplace() == "Yes":
|
||||
replacetext="Replaceable"
|
||||
else:
|
||||
replacetext="Not Replaceable"
|
||||
line="%s/%s/%s" % (self.getaudience(), self.getstability(), replacetext)
|
||||
return(line)
|
||||
|
||||
def getdocpage(self):
|
||||
line="### `%s`\n\n"\
|
||||
"* Synopsis\n\n"\
|
||||
"```\n%s\n"\
|
||||
"```\n\n" \
|
||||
"* Description\n\n" \
|
||||
"%s\n\n" \
|
||||
"* Returns\n\n" \
|
||||
"%s\n\n" \
|
||||
"| Classification | Level |\n" \
|
||||
"| :--- | :--- |\n" \
|
||||
"| Audience | %s |\n" \
|
||||
"| Stability | %s |\n" \
|
||||
"| Replaceable | %s |\n\n" \
|
||||
% (self.getname(),
|
||||
self.getusage(),
|
||||
self.getdesc(),
|
||||
self.getreturn(),
|
||||
self.getaudience(),
|
||||
self.getstability(),
|
||||
self.getreplace())
|
||||
return line
|
||||
|
||||
def __str__(self):
|
||||
line="{%s %s %s %s}" \
|
||||
% (self.getname(),
|
||||
self.getaudience(),
|
||||
self.getstability(),
|
||||
self.getreplace())
|
||||
return line
|
||||
|
||||
def main():
|
||||
parser=OptionParser(usage="usage: %prog --skipprnorep --output OUTFILE --input INFILE [--input INFILE ...]")
|
||||
parser.add_option("-o","--output", dest="outfile",
|
||||
action="store", type="string",
|
||||
help="file to create", metavar="OUTFILE")
|
||||
parser.add_option("-i","--input", dest="infile",
|
||||
action="append", type="string",
|
||||
help="file to read", metavar="INFILE")
|
||||
parser.add_option("--skipprnorep", dest="skipprnorep",
|
||||
action="store_true", help="Skip Private & Not Replaceable")
|
||||
|
||||
(options, args)=parser.parse_args()
|
||||
|
||||
allfuncs=[]
|
||||
for filename in options.infile:
|
||||
with open(filename,"r") as shellcode:
|
||||
funcdef=ShellFunction()
|
||||
for line in shellcode:
|
||||
if line.startswith('## @description'):
|
||||
funcdef.adddesc(line)
|
||||
elif line.startswith('## @audience'):
|
||||
funcdef.setaudience(line)
|
||||
elif line.startswith('## @stability'):
|
||||
funcdef.setstability(line)
|
||||
elif line.startswith('## @replaceable'):
|
||||
funcdef.setreplace(line)
|
||||
elif line.startswith('## @param'):
|
||||
funcdef.addparam(line)
|
||||
elif line.startswith('## @return'):
|
||||
funcdef.addreturn(line)
|
||||
elif line.startswith('function'):
|
||||
funcdef.setname(line)
|
||||
if options.skipprnorep and \
|
||||
funcdef.getaudience() == "Private" and \
|
||||
funcdef.getreplace() == "No":
|
||||
pass
|
||||
else:
|
||||
allfuncs.append(funcdef)
|
||||
funcdef=ShellFunction()
|
||||
|
||||
allfuncs=sorted(allfuncs)
|
||||
|
||||
outfile=open(options.outfile, "w")
|
||||
outfile.write(asflicense)
|
||||
for line in toc(allfuncs):
|
||||
outfile.write(line)
|
||||
|
||||
outfile.write("\n------\n\n")
|
||||
|
||||
header=[]
|
||||
for funcs in allfuncs:
|
||||
if header != funcs.getinter():
|
||||
header=funcs.getinter()
|
||||
line="## %s\n" % (funcs.headerbuild())
|
||||
outfile.write(line)
|
||||
outfile.write(funcs.getdocpage())
|
||||
outfile.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# Determine if the git diff patch file has prefixes.
|
||||
# These files are generated via "git diff" *without* the --no-prefix option.
|
||||
#
|
||||
# We can apply these patches more easily because we know that the a/ and b/
|
||||
# prefixes in the "diff" lines stands for the project root directory.
|
||||
# So we don't have to hunt for the project root.
|
||||
# And of course, we know that the patch file was generated using git, so we
|
||||
# know git apply can handle it properly.
|
||||
#
|
||||
# Arguments: git diff file name.
|
||||
# Return: 0 if it is a git diff with prefix; 1 otherwise.
|
||||
#
|
||||
has_prefix() {
|
||||
awk '/^diff --git / { if ($3 !~ "^a/" || $4 !~ "^b/") { exit 1 } }
|
||||
/^\+{3}|-{3} / { if ($2 !~ "^[ab]/" && $2 !~ "^/dev/null") { exit 1 } }' "$1"
|
||||
return $?
|
||||
}
|
||||
|
||||
PATCH_FILE=$1
|
||||
DRY_RUN=$2
|
||||
if [ -z "$PATCH_FILE" ]; then
|
||||
echo usage: $0 patch-file
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMPDIR=${TMPDIR:-/tmp}
|
||||
PATCH=${PATCH:-patch} # allow overriding patch binary
|
||||
|
||||
# Cleanup handler for temporary files
|
||||
TOCLEAN=""
|
||||
cleanup() {
|
||||
rm $TOCLEAN
|
||||
exit $1
|
||||
}
|
||||
trap "cleanup 1" HUP INT QUIT TERM
|
||||
|
||||
# Allow passing "-" for stdin patches
|
||||
if [ "$PATCH_FILE" == "-" ]; then
|
||||
PATCH_FILE="$TMPDIR/smart-apply.in.$RANDOM"
|
||||
cat /dev/fd/0 > $PATCH_FILE
|
||||
TOCLEAN="$TOCLEAN $PATCH_FILE"
|
||||
fi
|
||||
|
||||
ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
|
||||
if [[ ${PATCH_FILE} =~ ^http || ${PATCH_FILE} =~ ${ISSUE_RE} ]]; then
|
||||
# Allow downloading of patches
|
||||
PFILE="$TMPDIR/smart-apply.in.$RANDOM"
|
||||
TOCLEAN="$TOCLEAN $PFILE"
|
||||
if [[ ${PATCH_FILE} =~ ^http ]]; then
|
||||
patchURL="${PATCH_FILE}"
|
||||
else # Get URL of patch from JIRA
|
||||
wget -q -O "${PFILE}" "http://issues.apache.org/jira/browse/${PATCH_FILE}"
|
||||
if [[ $? != 0 ]]; then
|
||||
echo "Unable to determine what ${PATCH_FILE} may reference." 1>&2
|
||||
cleanup 1
|
||||
elif [[ $(grep -c 'Patch Available' "${PFILE}") == 0 ]]; then
|
||||
echo "${PATCH_FILE} is not \"Patch Available\". Exiting." 1>&2
|
||||
cleanup 1
|
||||
fi
|
||||
relativePatchURL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PFILE}" | grep -v -e 'htm[l]*$' | sort | tail -1 | grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
|
||||
patchURL="http://issues.apache.org${relativePatchURL}"
|
||||
fi
|
||||
if [[ -n $DRY_RUN ]]; then
|
||||
echo "Downloading ${patchURL}"
|
||||
fi
|
||||
wget -q -O "${PFILE}" "${patchURL}"
|
||||
if [[ $? != 0 ]]; then
|
||||
echo "${PATCH_FILE} could not be downloaded." 1>&2
|
||||
cleanup 1
|
||||
fi
|
||||
PATCH_FILE="${PFILE}"
|
||||
fi
|
||||
|
||||
# Case for git-diff patches
|
||||
if grep -q "^diff --git" "${PATCH_FILE}"; then
|
||||
GIT_FLAGS="--binary -v"
|
||||
if has_prefix "$PATCH_FILE"; then
|
||||
GIT_FLAGS="$GIT_FLAGS -p1"
|
||||
else
|
||||
GIT_FLAGS="$GIT_FLAGS -p0"
|
||||
fi
|
||||
if [[ -z $DRY_RUN ]]; then
|
||||
GIT_FLAGS="$GIT_FLAGS --stat --apply"
|
||||
echo Going to apply git patch with: git apply "${GIT_FLAGS}"
|
||||
else
|
||||
GIT_FLAGS="$GIT_FLAGS --check"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
git apply ${GIT_FLAGS} "${PATCH_FILE}"
|
||||
if [[ $? == 0 ]]; then
|
||||
cleanup 0
|
||||
fi
|
||||
echo "git apply failed. Going to apply the patch with: ${PATCH}"
|
||||
fi
|
||||
|
||||
# Come up with a list of changed files into $TMP
|
||||
TMP="$TMPDIR/smart-apply.paths.$RANDOM"
|
||||
TOCLEAN="$TOCLEAN $TMP"
|
||||
|
||||
if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
|
||||
PLEVEL=0
|
||||
#if the patch applied at P0 there is the possability that all we are doing
|
||||
# is adding new files and they would apply anywhere. So try to guess the
|
||||
# correct place to put those files.
|
||||
|
||||
TMP2="$TMPDIR/smart-apply.paths.2.$RANDOM"
|
||||
TOCLEAN="$TOCLEAN $TMP2"
|
||||
|
||||
egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort -u > $TMP2
|
||||
|
||||
if [ ! -s $TMP2 ]; then
|
||||
echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
|
||||
cleanup 1
|
||||
fi
|
||||
|
||||
#first off check that all of the files do not exist
|
||||
FOUND_ANY=0
|
||||
for CHECK_FILE in $(cat $TMP2)
|
||||
do
|
||||
if [[ -f $CHECK_FILE ]]; then
|
||||
FOUND_ANY=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$FOUND_ANY" = "0" ]]; then
|
||||
#all of the files are new files so we have to guess where the correct place to put it is.
|
||||
|
||||
# if all of the lines start with a/ or b/, then this is a git patch that
|
||||
# was generated without --no-prefix
|
||||
if ! grep -qv '^a/\|^b/' $TMP2 ; then
|
||||
echo Looks like this is a git patch. Stripping a/ and b/ prefixes
|
||||
echo and incrementing PLEVEL
|
||||
PLEVEL=$[$PLEVEL + 1]
|
||||
sed -i -e 's,^[ab]/,,' $TMP2
|
||||
fi
|
||||
|
||||
PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 $TMP2 | sort -u)
|
||||
|
||||
# if we are at the project root then nothing more to do
|
||||
if [[ -d hadoop-common-project ]]; then
|
||||
echo Looks like this is being run at project root
|
||||
|
||||
# if all of the lines start with hadoop-common/, hadoop-hdfs/, hadoop-yarn/ or hadoop-mapreduce/, this is
|
||||
# relative to the hadoop root instead of the subproject root, so we need
|
||||
# to chop off another layer
|
||||
elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-yarn-project|hadoop-mapreduce-project)$ ]]; then
|
||||
|
||||
echo Looks like this is relative to project root. Increasing PLEVEL
|
||||
PLEVEL=$[$PLEVEL + 1]
|
||||
|
||||
elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-yarn-project\|hadoop-mapreduce-project' ; then
|
||||
echo Looks like this is a cross-subproject patch. Try applying from the project root
|
||||
cleanup 1
|
||||
fi
|
||||
fi
|
||||
elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
|
||||
PLEVEL=1
|
||||
elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
|
||||
PLEVEL=2
|
||||
else
|
||||
echo "The patch does not appear to apply with p0 to p2";
|
||||
cleanup 1;
|
||||
fi
|
||||
|
||||
# If this is a dry run then exit instead of applying the patch
|
||||
if [[ -n $DRY_RUN ]]; then
|
||||
cleanup 0;
|
||||
fi
|
||||
|
||||
echo Going to apply patch with: $PATCH -p$PLEVEL
|
||||
$PATCH -p$PLEVEL -E < $PATCH_FILE
|
||||
|
||||
cleanup $?
|
||||
|
|
@ -1,205 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
add_plugin checkstyle
|
||||
|
||||
CHECKSTYLE_TIMER=0
|
||||
|
||||
# if it ends in an explicit .sh, then this is shell code.
|
||||
# if it doesn't have an extension, we assume it is shell code too
|
||||
function checkstyle_filefilter
|
||||
{
|
||||
local filename=$1
|
||||
|
||||
if [[ ${filename} =~ \.java$ ]]; then
|
||||
add_test checkstyle
|
||||
fi
|
||||
}
|
||||
|
||||
function checkstyle_mvnrunner
|
||||
{
|
||||
local logfile=$1
|
||||
local output=$2
|
||||
local tmp=${PATCH_DIR}/$$.${RANDOM}
|
||||
local j
|
||||
|
||||
"${MVN}" clean test checkstyle:checkstyle -DskipTests \
|
||||
-Dcheckstyle.consoleOutput=true \
|
||||
"-D${PROJECT_NAME}PatchProcess" 2>&1 \
|
||||
| tee "${logfile}" \
|
||||
| ${GREP} ^/ \
|
||||
| ${SED} -e "s,${BASEDIR},.,g" \
|
||||
> "${tmp}"
|
||||
|
||||
# the checkstyle output files are massive, so
|
||||
# let's reduce the work by filtering out files
|
||||
# that weren't changed. Some modules are
|
||||
# MASSIVE and this can cut the output down to
|
||||
# by orders of magnitude!!
|
||||
for j in ${CHANGED_FILES}; do
|
||||
${GREP} "${j}" "${tmp}" >> "${output}"
|
||||
done
|
||||
|
||||
rm "${tmp}" 2>/dev/null
|
||||
}
|
||||
|
||||
function checkstyle_preapply
|
||||
{
|
||||
local module_suffix
|
||||
local modules=${CHANGED_MODULES}
|
||||
local module
|
||||
|
||||
verify_needed_test checkstyle
|
||||
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "checkstyle plugin: prepatch"
|
||||
|
||||
start_clock
|
||||
|
||||
for module in ${modules}
|
||||
do
|
||||
pushd "${module}" >/dev/null
|
||||
echo " Running checkstyle in ${module}"
|
||||
module_suffix=$(basename "${module}")
|
||||
|
||||
checkstyle_mvnrunner \
|
||||
"${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
|
||||
|
||||
if [[ $? != 0 ]] ; then
|
||||
echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
|
||||
add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
|
||||
fi
|
||||
popd >/dev/null
|
||||
done
|
||||
|
||||
# keep track of how much as elapsed for us already
|
||||
CHECKSTYLE_TIMER=$(stop_clock)
|
||||
return 0
|
||||
}
|
||||
|
||||
function checkstyle_calcdiffs
|
||||
{
|
||||
local orig=$1
|
||||
local new=$2
|
||||
local diffout=$3
|
||||
local tmp=${PATCH_DIR}/cs.$$.${RANDOM}
|
||||
local count=0
|
||||
local j
|
||||
|
||||
# first, pull out just the errors
|
||||
# shellcheck disable=SC2016
|
||||
${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
|
||||
|
||||
# compare the errors, generating a string of line
|
||||
# numbers. Sorry portability: GNU diff makes this too easy
|
||||
${DIFF} --unchanged-line-format="" \
|
||||
--old-line-format="" \
|
||||
--new-line-format="%dn " \
|
||||
"${tmp}.branch" \
|
||||
"${tmp}.patch" > "${tmp}.lined"
|
||||
|
||||
# now, pull out those lines of the raw output
|
||||
# shellcheck disable=SC2013
|
||||
for j in $(cat "${tmp}.lined"); do
|
||||
# shellcheck disable=SC2086
|
||||
head -${j} "${new}" | tail -1 >> "${diffout}"
|
||||
done
|
||||
|
||||
if [[ -f "${diffout}" ]]; then
|
||||
# shellcheck disable=SC2016
|
||||
count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
|
||||
fi
|
||||
rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
|
||||
echo "${count}"
|
||||
}
|
||||
|
||||
function checkstyle_postapply
|
||||
{
|
||||
local rc=0
|
||||
local module
|
||||
local modules=${CHANGED_MODULES}
|
||||
local module_suffix
|
||||
local numprepatch=0
|
||||
local numpostpatch=0
|
||||
local diffpostpatch=0
|
||||
|
||||
verify_needed_test checkstyle
|
||||
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "checkstyle plugin: postpatch"
|
||||
|
||||
start_clock
|
||||
|
||||
# add our previous elapsed to our new timer
|
||||
# by setting the clock back
|
||||
offset_clock "${CHECKSTYLE_TIMER}"
|
||||
|
||||
for module in ${modules}
|
||||
do
|
||||
pushd "${module}" >/dev/null
|
||||
echo " Running checkstyle in ${module}"
|
||||
module_suffix=$(basename "${module}")
|
||||
|
||||
checkstyle_mvnrunner \
|
||||
"${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
|
||||
|
||||
if [[ $? != 0 ]] ; then
|
||||
((rc = rc +1))
|
||||
echo "Post-patch checkstyle compilation is broken."
|
||||
add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
|
||||
continue
|
||||
fi
|
||||
|
||||
#shellcheck disable=SC2016
|
||||
diffpostpatch=$(checkstyle_calcdiffs \
|
||||
"${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
|
||||
"${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
|
||||
|
||||
if [[ ${diffpostpatch} -gt 0 ]] ; then
|
||||
((rc = rc + 1))
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
|
||||
# shellcheck disable=SC2016
|
||||
numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
|
||||
|
||||
add_jira_table -1 checkstyle "The applied patch generated "\
|
||||
"${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
|
||||
footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
|
||||
fi
|
||||
|
||||
popd >/dev/null
|
||||
done
|
||||
|
||||
if [[ ${rc} -gt 0 ]] ; then
|
||||
add_jira_footer checkstyle "${footer}"
|
||||
return 1
|
||||
fi
|
||||
add_jira_table +1 checkstyle "There were no new checkstyle issues."
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1,178 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
add_plugin shellcheck
|
||||
|
||||
SHELLCHECK_TIMER=0
|
||||
|
||||
SHELLCHECK=${SHELLCHECK:-$(which shellcheck 2>/dev/null)}
|
||||
|
||||
SHELLCHECK_SPECIFICFILES=""
|
||||
|
||||
# if it ends in an explicit .sh, then this is shell code.
|
||||
# if it doesn't have an extension, we assume it is shell code too
|
||||
function shellcheck_filefilter
|
||||
{
|
||||
local filename=$1
|
||||
|
||||
if [[ ${filename} =~ \.sh$ ]]; then
|
||||
add_test shellcheck
|
||||
SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
|
||||
fi
|
||||
|
||||
if [[ ! ${filename} =~ \. ]]; then
|
||||
add_test shellcheck
|
||||
fi
|
||||
}
|
||||
|
||||
function shellcheck_private_findbash
|
||||
{
|
||||
local i
|
||||
local value
|
||||
local list
|
||||
|
||||
while read line; do
|
||||
value=$(find "${line}" ! -name '*.cmd' -type f \
|
||||
| ${GREP} -E -v '(.orig$|.rej$)')
|
||||
list="${list} ${value}"
|
||||
done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
|
||||
# shellcheck disable=SC2086
|
||||
echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
|
||||
}
|
||||
|
||||
function shellcheck_preapply
|
||||
{
|
||||
local i
|
||||
|
||||
verify_needed_test shellcheck
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "shellcheck plugin: prepatch"
|
||||
|
||||
if [[ ! -x "${SHELLCHECK}" ]]; then
|
||||
hadoop_error "shellcheck is not available."
|
||||
return 0
|
||||
fi
|
||||
|
||||
start_clock
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
|
||||
|
||||
echo "Running shellcheck against all identifiable shell scripts"
|
||||
pushd "${BASEDIR}" >/dev/null
|
||||
for i in $(shellcheck_private_findbash); do
|
||||
if [[ -f ${i} ]]; then
|
||||
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
|
||||
fi
|
||||
done
|
||||
popd > /dev/null
|
||||
# keep track of how much as elapsed for us already
|
||||
SHELLCHECK_TIMER=$(stop_clock)
|
||||
return 0
|
||||
}
|
||||
|
||||
function shellcheck_calcdiffs
|
||||
{
|
||||
local orig=$1
|
||||
local new=$2
|
||||
local diffout=$3
|
||||
local tmp=${PATCH_DIR}/sc.$$.${RANDOM}
|
||||
local count=0
|
||||
local j
|
||||
|
||||
# first, pull out just the errors
|
||||
# shellcheck disable=SC2016
|
||||
${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
|
||||
|
||||
# compare the errors, generating a string of line
|
||||
# numbers. Sorry portability: GNU diff makes this too easy
|
||||
${DIFF} --unchanged-line-format="" \
|
||||
--old-line-format="" \
|
||||
--new-line-format="%dn " \
|
||||
"${tmp}.branch" \
|
||||
"${tmp}.patch" > "${tmp}.lined"
|
||||
|
||||
# now, pull out those lines of the raw output
|
||||
# shellcheck disable=SC2013
|
||||
for j in $(cat "${tmp}.lined"); do
|
||||
# shellcheck disable=SC2086
|
||||
head -${j} "${new}" | tail -1 >> "${diffout}"
|
||||
done
|
||||
|
||||
if [[ -f "${diffout}" ]]; then
|
||||
# shellcheck disable=SC2016
|
||||
count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
|
||||
fi
|
||||
rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
|
||||
echo "${count}"
|
||||
}
|
||||
|
||||
function shellcheck_postapply
|
||||
{
|
||||
local i
|
||||
|
||||
verify_needed_test shellcheck
|
||||
if [[ $? == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
big_console_header "shellcheck plugin: postpatch"
|
||||
|
||||
if [[ ! -x "${SHELLCHECK}" ]]; then
|
||||
hadoop_error "shellcheck is not available."
|
||||
add_jira_table 0 shellcheck "Shellcheck was not available."
|
||||
return 0
|
||||
fi
|
||||
|
||||
start_clock
|
||||
|
||||
# add our previous elapsed to our new timer
|
||||
# by setting the clock back
|
||||
offset_clock "${SHELLCHECK_TIMER}"
|
||||
|
||||
echo "Running shellcheck against all identifiable shell scripts"
|
||||
# we re-check this in case one has been added
|
||||
for i in $(shellcheck_private_findbash); do
|
||||
${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
|
||||
done
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
|
||||
# shellcheck disable=SC2016
|
||||
numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
|
||||
|
||||
diffPostpatch=$(shellcheck_calcdiffs \
|
||||
"${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
|
||||
"${PATCH_DIR}/patchshellcheck-result.txt" \
|
||||
"${PATCH_DIR}/diffpatchshellcheck.txt"
|
||||
)
|
||||
|
||||
if [[ ${diffPostpatch} -gt 0 ]] ; then
|
||||
add_jira_table -1 shellcheck "The applied patch generated "\
|
||||
"${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
|
||||
add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
|
||||
return 1
|
||||
fi
|
||||
|
||||
add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
add_plugin whitespace
|
||||
|
||||
function whitespace_postapply
|
||||
{
|
||||
local count
|
||||
local j
|
||||
|
||||
big_console_header "Checking for whitespace at the end of lines"
|
||||
start_clock
|
||||
|
||||
pushd "${BASEDIR}" >/dev/null
|
||||
for j in ${CHANGED_FILES}; do
|
||||
${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
|
||||
done
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
|
||||
|
||||
if [[ ${count} -gt 0 ]]; then
|
||||
add_jira_table -1 whitespace "The patch has ${count}"\
|
||||
" line(s) that end in whitespace. Use git apply --whitespace=fix."
|
||||
add_jira_footer whitespace "@@BASE@@/whitespace.txt"
|
||||
popd >/dev/null
|
||||
return 1
|
||||
fi
|
||||
|
||||
popd >/dev/null
|
||||
add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
|
||||
return 0
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -513,10 +513,9 @@
|
|||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>python</executable>
|
||||
<executable>${basedir}/../../dev-support/bin/shelldocs</executable>
|
||||
<workingDirectory>src/site/markdown</workingDirectory>
|
||||
<arguments>
|
||||
<argument>${basedir}/../../dev-support/shelldocs.py</argument>
|
||||
<argument>--skipprnorep</argument>
|
||||
<argument>--output</argument>
|
||||
<argument>${basedir}/src/site/markdown/UnixShellAPI.md</argument>
|
||||
|
|
@ -982,11 +981,10 @@
|
|||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>python</executable>
|
||||
<executable>${basedir}/../../dev-support/bin/releasedocmaker</executable>
|
||||
<workingDirectory>src/site/markdown/release/</workingDirectory>
|
||||
<requiresOnline>true</requiresOnline>
|
||||
<arguments>
|
||||
<argument>${basedir}/../../dev-support/releasedocmaker.py</argument>
|
||||
<argument>--index</argument>
|
||||
<argument>--license</argument>
|
||||
<argument>--project</argument>
|
||||
|
|
|
|||
Loading…
Reference in New Issue