mirror of https://github.com/apache/lucene.git
330 lines
11 KiB
Python
330 lines
11 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import datetime
|
|
import re
|
|
import time
|
|
import shutil
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import textwrap
|
|
import urllib.request, urllib.error, urllib.parse
|
|
import xml.etree.ElementTree as ET
|
|
|
|
LOG = '/tmp/release.log'
|
|
|
|
def log(msg):
|
|
f = open(LOG, mode='ab')
|
|
f.write(msg.encode('utf-8'))
|
|
f.close()
|
|
|
|
def run(command):
|
|
log('\n\n%s: RUN: %s\n' % (datetime.datetime.now(), command))
|
|
if os.system('%s >> %s 2>&1' % (command, LOG)):
|
|
msg = ' FAILED: %s [see log %s]' % (command, LOG)
|
|
print(msg)
|
|
raise RuntimeError(msg)
|
|
|
|
def runAndSendGPGPassword(command, password):
|
|
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
|
|
f = open(LOG, 'ab')
|
|
while True:
|
|
p.stdout.flush()
|
|
line = p.stdout.readline()
|
|
if len(line) == 0:
|
|
break
|
|
f.write(line)
|
|
if line.find(b'Enter GPG keystore password:') != -1:
|
|
time.sleep(1.0)
|
|
p.stdin.write((password + '\n').encode('UTF-8'))
|
|
p.stdin.write('\n'.encode('UTF-8'))
|
|
|
|
try:
|
|
result = p.wait(timeout=120)
|
|
if result != 0:
|
|
msg = ' FAILED: %s [see log %s]' % (command, LOG)
|
|
print(msg)
|
|
raise RuntimeError(msg)
|
|
except TimeoutExpired:
|
|
msg = ' FAILED: %s [timed out after 2 minutes; see log %s]' % (command, LOG)
|
|
print(msg)
|
|
raise RuntimeError(msg)
|
|
|
|
def load(urlString):
|
|
try:
|
|
content = urllib.request.urlopen(urlString).read().decode('utf-8')
|
|
except Exception as e:
|
|
print('Retrying download of url %s after exception: %s' % (urlString, e))
|
|
content = urllib.request.urlopen(urlString).read().decode('utf-8')
|
|
return content
|
|
|
|
def getGitRev():
|
|
status = os.popen('git status').read().strip()
|
|
if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
|
|
raise RuntimeError('git clone is dirty:\n\n%s' % status)
|
|
branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip()
|
|
command = 'git log origin/%s..' % branch
|
|
unpushedCommits = os.popen(command).read().strip()
|
|
if len(unpushedCommits) > 0:
|
|
raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, unpushedCommits))
|
|
|
|
print(' git clone is clean')
|
|
return os.popen('git rev-parse HEAD').read().strip()
|
|
|
|
def prepare(root, version, gpgKeyID, gpgPassword):
|
|
print()
|
|
print('Prepare release...')
|
|
if os.path.exists(LOG):
|
|
os.remove(LOG)
|
|
|
|
os.chdir(root)
|
|
print(' git pull...')
|
|
run('git pull')
|
|
|
|
rev = getGitRev()
|
|
print(' git rev: %s' % rev)
|
|
log('\nGIT rev: %s\n' % rev)
|
|
|
|
print(' Check DOAP files')
|
|
checkDOAPfiles(version)
|
|
|
|
print(' ant clean test validate documentation-lint')
|
|
run('ant clean test validate documentation-lint')
|
|
|
|
open('rev.txt', mode='wb').write(rev.encode('UTF-8'))
|
|
|
|
print(' lucene prepare-release')
|
|
os.chdir('lucene')
|
|
cmd = 'ant -Dversion=%s' % version
|
|
if gpgKeyID is not None:
|
|
cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID
|
|
else:
|
|
cmd += ' prepare-release-no-sign'
|
|
|
|
if gpgPassword is not None:
|
|
runAndSendGPGPassword(cmd, gpgPassword)
|
|
else:
|
|
run(cmd)
|
|
|
|
print(' solr prepare-release')
|
|
os.chdir('../solr')
|
|
cmd = 'ant -Dversion=%s' % version
|
|
if gpgKeyID is not None:
|
|
cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID
|
|
else:
|
|
cmd += ' prepare-release-no-sign'
|
|
|
|
if gpgPassword is not None:
|
|
runAndSendGPGPassword(cmd, gpgPassword)
|
|
else:
|
|
run(cmd)
|
|
|
|
print(' done!')
|
|
print()
|
|
return rev
|
|
|
|
reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE)
|
|
reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE)
|
|
reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE)
|
|
def checkDOAPfiles(version):
|
|
# In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced.
|
|
errorMessages = []
|
|
for product in 'lucene', 'solr':
|
|
url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product)
|
|
distpage = load(url)
|
|
releases = set()
|
|
for regex in reVersion1, reVersion2:
|
|
for tup in regex.findall(distpage):
|
|
if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases
|
|
continue
|
|
releases.add(normalizeVersion(tup))
|
|
doapNS = '{http://usefulinc.com/ns/doap#}'
|
|
xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS)
|
|
doapFile = "dev-tools/doap/%s.rdf" % product
|
|
treeRoot = ET.parse(doapFile).getroot()
|
|
doapRevisions = set()
|
|
for revision in treeRoot.findall(xpathRevision):
|
|
match = reDoapRevision.match(revision.text)
|
|
if (match is not None):
|
|
if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
|
|
doapRevisions.add(normalizeVersion(match.groups()))
|
|
else:
|
|
errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile))
|
|
missingDoapRevisions = set()
|
|
for release in releases:
|
|
if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
|
|
missingDoapRevisions.add(release)
|
|
if len(missingDoapRevisions) > 0:
|
|
errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions))))
|
|
if (len(errorMessages) > 0):
|
|
raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)'
|
|
% '\n'.join(errorMessages))
|
|
|
|
def normalizeVersion(tup):
|
|
suffix = ''
|
|
if tup[-1] is not None and tup[-1].lower() == '-alpha':
|
|
tup = tup[:(len(tup) - 1)]
|
|
suffix = '-ALPHA'
|
|
elif tup[-1] is not None and tup[-1].lower() == '-beta':
|
|
tup = tup[:(len(tup) - 1)]
|
|
suffix = '-BETA'
|
|
while tup[-1] in ('', None):
|
|
tup = tup[:(len(tup) - 1)]
|
|
while len(tup) < 3:
|
|
tup = tup + ('0',)
|
|
return '.'.join(tup) + suffix
|
|
|
|
def pushLocal(version, root, rev, rcNum, localDir):
|
|
print('Push local [%s]...' % localDir)
|
|
os.makedirs(localDir)
|
|
|
|
dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev)
|
|
os.makedirs('%s/%s/lucene' % (localDir, dir))
|
|
os.makedirs('%s/%s/solr' % (localDir, dir))
|
|
print(' Lucene')
|
|
os.chdir('%s/lucene/dist' % root)
|
|
print(' zip...')
|
|
if os.path.exists('lucene.tar.bz2'):
|
|
os.remove('lucene.tar.bz2')
|
|
run('tar cjf lucene.tar.bz2 *')
|
|
|
|
os.chdir('%s/%s/lucene' % (localDir, dir))
|
|
print(' unzip...')
|
|
run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root)
|
|
os.remove('%s/lucene/dist/lucene.tar.bz2' % root)
|
|
|
|
print(' Solr')
|
|
os.chdir('%s/solr/package' % root)
|
|
print(' zip...')
|
|
if os.path.exists('solr.tar.bz2'):
|
|
os.remove('solr.tar.bz2')
|
|
run('tar cjf solr.tar.bz2 *')
|
|
print(' unzip...')
|
|
os.chdir('%s/%s/solr' % (localDir, dir))
|
|
run('tar xjf "%s/solr/package/solr.tar.bz2"' % root)
|
|
os.remove('%s/solr/package/solr.tar.bz2' % root)
|
|
|
|
print(' KEYS')
|
|
run('wget http://home.apache.org/keys/group/lucene.asc')
|
|
os.rename('lucene.asc', 'KEYS')
|
|
run('chmod a+r-w KEYS')
|
|
run('cp KEYS ../lucene')
|
|
|
|
print(' chmod...')
|
|
os.chdir('..')
|
|
run('chmod -R a+rX-w .')
|
|
|
|
print(' done!')
|
|
return 'file://%s/%s' % (os.path.abspath(localDir), dir)
|
|
|
|
def read_version(path):
|
|
version_props_file = os.path.join(path, 'lucene', 'version.properties')
|
|
return re.search(r'version\.base=(.*)', open(version_props_file).read()).group(1)
|
|
|
|
def parse_config():
|
|
epilogue = textwrap.dedent('''
|
|
Example usage for a Release Manager:
|
|
python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
|
|
''')
|
|
description = 'Utility to build, push, and test a release.'
|
|
parser = argparse.ArgumentParser(description=description, epilog=epilogue,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument('--no-prepare', dest='prepare', default=True, action='store_false',
|
|
help='Use the already built release in the provided checkout')
|
|
parser.add_argument('--push-local', metavar='PATH',
|
|
help='Push the release to the local path')
|
|
parser.add_argument('--sign', metavar='KEYID',
|
|
help='Sign the release with the given gpg key')
|
|
parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
|
|
help='Release Candidate number. Default: 1')
|
|
parser.add_argument('--root', metavar='PATH', default='.',
|
|
help='Root of Git working tree for lucene-solr. Default: "." (the current directory)')
|
|
config = parser.parse_args()
|
|
|
|
if not config.prepare and config.sign:
|
|
parser.error('Cannot sign already built release')
|
|
if config.push_local is not None and os.path.exists(config.push_local):
|
|
parser.error('Cannot push to local path that already exists')
|
|
if config.rc_num <= 0:
|
|
parser.error('Release Candidate number must be a positive integer')
|
|
if not os.path.isdir(config.root):
|
|
parser.error('Root path "%s" is not a directory' % config.root)
|
|
cwd = os.getcwd()
|
|
os.chdir(config.root)
|
|
config.root = os.getcwd() # Absolutize root dir
|
|
if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]):
|
|
parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root)
|
|
os.chdir(cwd)
|
|
|
|
config.version = read_version(config.root)
|
|
print('Building version: %s' % config.version)
|
|
|
|
if config.sign:
|
|
sys.stdout.flush()
|
|
import getpass
|
|
config.key_id = config.sign
|
|
config.key_password = getpass.getpass('Enter GPG keystore password: ')
|
|
else:
|
|
config.gpg_password = None
|
|
|
|
return config
|
|
|
|
def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
|
|
if os.system('git --version >/dev/null 2>/dev/null'):
|
|
raise RuntimeError('"git --version" returned a non-zero exit code.')
|
|
check_ant()
|
|
|
|
def check_ant():
|
|
antVersion = os.popen('ant -version').read().strip()
|
|
if (antVersion.startswith('Apache Ant(TM) version 1.8')):
|
|
return
|
|
if (antVersion.startswith('Apache Ant(TM) version 1.9')):
|
|
return
|
|
if (antVersion.startswith('Apache Ant(TM) version 1.10')):
|
|
return
|
|
raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
|
|
|
|
def main():
|
|
check_cmdline_tools()
|
|
|
|
c = parse_config()
|
|
|
|
if c.prepare:
|
|
rev = prepare(c.root, c.version, c.key_id, c.key_password)
|
|
else:
|
|
os.chdir(c.root)
|
|
rev = open('rev.txt', encoding='UTF-8').read()
|
|
|
|
if c.push_local:
|
|
url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local)
|
|
else:
|
|
url = None
|
|
|
|
if url is not None:
|
|
print(' URL: %s' % url)
|
|
print('Next run the smoker tester:')
|
|
p = re.compile(".*/")
|
|
m = p.match(sys.argv[0])
|
|
print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url))
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
print('Keyboard interrupt...exiting')
|
|
|