Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
lucene/dev-tools/scripts/buildAndPushRelease.py

354 lines
13 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import datetime
import re
import time
import os
import sys
import subprocess
from subprocess import TimeoutExpired
import textwrap
import urllib.request, urllib.error, urllib.parse
import xml.etree.ElementTree as ET
LOG = '/tmp/release.log'
def log(msg):
f = open(LOG, mode='ab')
f.write(msg.encode('utf-8'))
f.close()
def run(command):
log('\n\n%s: RUN: %s\n' % (datetime.datetime.now(), command))
if os.system('%s >> %s 2>&1' % (command, LOG)):
msg = ' FAILED: %s [see log %s]' % (command, LOG)
print(msg)
raise RuntimeError(msg)
def runAndSendGPGPassword(command, password):
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
f = open(LOG, 'ab')
while True:
p.stdout.flush()
line = p.stdout.readline()
if len(line) == 0:
break
f.write(line)
if line.find(b'Enter GPG keystore password:') != -1:
time.sleep(1.0)
p.stdin.write((password + '\n').encode('UTF-8'))
p.stdin.write('\n'.encode('UTF-8'))
try:
result = p.wait(timeout=120)
if result != 0:
msg = ' FAILED: %s [see log %s]' % (command, LOG)
print(msg)
raise RuntimeError(msg)
except TimeoutExpired:
msg = ' FAILED: %s [timed out after 2 minutes; see log %s]' % (command, LOG)
print(msg)
raise RuntimeError(msg)
def load(urlString, encoding="utf-8"):
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
print('Retrying download of url %s after exception: %s' % (urlString, e))
content = urllib.request.urlopen(urlString).read().decode(encoding)
return content
def getGitRev():
status = os.popen('git status').read().strip()
if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
raise RuntimeError('git clone is dirty:\n\n%s' % status)
branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip()
command = 'git log origin/%s..' % branch
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if len(stdout.strip()) > 0:
raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, stdout.decode('utf-8')))
if len(stderr.strip()) > 0:
raise RuntimeError('Command "%s" failed:\n\n%s' % (command, stderr.decode('utf-8')))
print(' git clone is clean')
return os.popen('git rev-parse HEAD').read().strip()
def prepare(root, version, gpgKeyID, gpgPassword):
print()
print('Prepare release...')
if os.path.exists(LOG):
os.remove(LOG)
os.chdir(root)
print(' git pull...')
run('git pull')
rev = getGitRev()
print(' git rev: %s' % rev)
log('\nGIT rev: %s\n' % rev)
print(' Check DOAP files')
checkDOAPfiles(version)
print(' ./gradlew -Dtests.badapples=false clean check')
run('./gradlew -Dtests.badapples=false clean check')
open('rev.txt', mode='wb').write(rev.encode('UTF-8'))
print(' prepare-release')
cmd = './gradlew -Dversion=%s clean' % version
# TODO cd lucene ; ant dist-all?
# TODO cd solr ; ant package?
if gpgKeyID is not None:
# TODO sign
# cmd += ' -Psigning.keyId=%s publishSignedPublicationToMavenLocal' % gpgKeyID
pass
cmd += ' mavenLocal'
if gpgPassword is not None:
runAndSendGPGPassword(cmd, gpgPassword)
else:
run(cmd)
print(' done!')
print()
return rev
reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE)
reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE)
reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE)
def checkDOAPfiles(version):
# In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced.
errorMessages = []
for product in 'lucene', 'solr':
url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product)
distpage = load(url)
releases = set()
for regex in reVersion1, reVersion2:
for tup in regex.findall(distpage):
if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases
continue
releases.add(normalizeVersion(tup))
doapNS = '{http://usefulinc.com/ns/doap#}'
xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS)
doapFile = "dev-tools/doap/%s.rdf" % product
treeRoot = ET.parse(doapFile).getroot()
doapRevisions = set()
for revision in treeRoot.findall(xpathRevision):
match = reDoapRevision.match(revision.text)
if (match is not None):
if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
doapRevisions.add(normalizeVersion(match.groups()))
else:
errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile))
missingDoapRevisions = set()
for release in releases:
if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
missingDoapRevisions.add(release)
if len(missingDoapRevisions) > 0:
errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions))))
if (len(errorMessages) > 0):
raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)'
% '\n'.join(errorMessages))
def normalizeVersion(tup):
suffix = ''
if tup[-1] is not None and tup[-1].lower() == '-alpha':
tup = tup[:(len(tup) - 1)]
suffix = '-ALPHA'
elif tup[-1] is not None and tup[-1].lower() == '-beta':
tup = tup[:(len(tup) - 1)]
suffix = '-BETA'
while tup[-1] in ('', None):
tup = tup[:(len(tup) - 1)]
while len(tup) < 3:
tup = tup + ('0',)
return '.'.join(tup) + suffix
def pushLocal(version, root, rev, rcNum, localDir):
print('Push local [%s]...' % localDir)
os.makedirs(localDir)
dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev)
os.makedirs('%s/%s/lucene' % (localDir, dir))
os.makedirs('%s/%s/solr' % (localDir, dir))
print(' Lucene')
os.chdir('%s/lucene/dist' % root)
print(' zip...')
if os.path.exists('lucene.tar.bz2'):
os.remove('lucene.tar.bz2')
run('tar cjf lucene.tar.bz2 *')
os.chdir('%s/%s/lucene' % (localDir, dir))
print(' unzip...')
run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root)
os.remove('%s/lucene/dist/lucene.tar.bz2' % root)
print(' Solr')
os.chdir('%s/solr/package' % root)
print(' zip...')
if os.path.exists('solr.tar.bz2'):
os.remove('solr.tar.bz2')
run('tar cjf solr.tar.bz2 *')
print(' unzip...')
os.chdir('%s/%s/solr' % (localDir, dir))
run('tar xjf "%s/solr/package/solr.tar.bz2"' % root)
os.remove('%s/solr/package/solr.tar.bz2' % root)
print(' chmod...')
os.chdir('..')
run('chmod -R a+rX-w .')
print(' done!')
return 'file://%s/%s' % (os.path.abspath(localDir), dir)
def read_version(path):
version_props_file = os.path.join(path, 'lucene', 'version.properties')
return re.search(r'version\.base=(.*)', open(version_props_file).read()).group(1)
def parse_config():
epilogue = textwrap.dedent('''
Example usage for a Release Manager:
python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
''')
description = 'Utility to build, push, and test a release.'
parser = argparse.ArgumentParser(description=description, epilog=epilogue,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--no-prepare', dest='prepare', default=True, action='store_false',
help='Use the already built release in the provided checkout')
parser.add_argument('--local-keys', metavar='PATH',
help='Uses local KEYS file to validate presence of RM\'s gpg key')
parser.add_argument('--push-local', metavar='PATH',
help='Push the release to the local path')
parser.add_argument('--sign', metavar='KEYID',
help='Sign the release with the given gpg key')
parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
help='Release Candidate number. Default: 1')
parser.add_argument('--root', metavar='PATH', default='.',
help='Root of Git working tree for lucene-solr. Default: "." (the current directory)')
parser.add_argument('--logfile', metavar='PATH',
help='Specify log file path (default /tmp/release.log)')
config = parser.parse_args()
if not config.prepare and config.sign:
parser.error('Cannot sign already built release')
if config.push_local is not None and os.path.exists(config.push_local):
parser.error('Cannot push to local path that already exists')
if config.rc_num <= 0:
parser.error('Release Candidate number must be a positive integer')
if not os.path.isdir(config.root):
parser.error('Root path "%s" is not a directory' % config.root)
if config.local_keys is not None and not os.path.exists(config.local_keys):
parser.error('Local KEYS file "%s" not found' % config.local_keys)
cwd = os.getcwd()
os.chdir(config.root)
config.root = os.getcwd() # Absolutize root dir
if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]):
parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root)
os.chdir(cwd)
global LOG
if config.logfile:
LOG = config.logfile
config.version = read_version(config.root)
print('Building version: %s' % config.version)
return config
def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
if os.system('git --version >/dev/null 2>/dev/null'):
raise RuntimeError('"git --version" returned a non-zero exit code.')
def check_key_in_keys(gpgKeyID, local_keys):
if gpgKeyID is not None:
print(' Verify your gpg key is in the main KEYS file')
if local_keys is not None:
print(" Using local KEYS file %s" % local_keys)
keysFileText = open(local_keys, encoding='iso-8859-1').read()
keysFileLocation = local_keys
else:
keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
keysFileLocation = keysFileURL
print(" Using online KEYS file %s" % keysFileURL)
keysFileText = load(keysFileURL, encoding='iso-8859-1')
if len(gpgKeyID) > 2 and gpgKeyID[0:2] == '0x':
gpgKeyID = gpgKeyID[2:]
if len(gpgKeyID) > 40:
gpgKeyID = gpgKeyID.replace(" ", "")
if len(gpgKeyID) == 8:
gpgKeyID8Char = "%s %s" % (gpgKeyID[0:4], gpgKeyID[4:8])
re_to_match = r"^pub .*\n\s+(\w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} %s|\w{32}%s)" % (gpgKeyID8Char, gpgKeyID)
elif len(gpgKeyID) == 40:
gpgKeyID40Char = "%s %s %s %s %s %s %s %s %s %s" % \
(gpgKeyID[0:4], gpgKeyID[4:8], gpgKeyID[8:12], gpgKeyID[12:16], gpgKeyID[16:20],
gpgKeyID[20:24], gpgKeyID[24:28], gpgKeyID[28:32], gpgKeyID[32:36], gpgKeyID[36:])
re_to_match = r"^pub .*\n\s+(%s|%s)" % (gpgKeyID40Char, gpgKeyID)
else:
print('Invalid gpg key id format [%s]. Must be 8 byte short ID or 40 byte fingerprint, with or without 0x prefix, no spaces.' % gpgKeyID)
exit(2)
if re.search(re_to_match, keysFileText, re.MULTILINE):
print(' Found key %s in KEYS file at %s' % (gpgKeyID, keysFileLocation))
else:
print(' ERROR: Did not find your key %s in KEYS file at %s. Please add it and try again.' % (gpgKeyID, keysFileLocation))
if local_keys is not None:
print(' You are using a local KEYS file. Make sure it is up to date or validate against the online version')
exit(2)
def main():
check_cmdline_tools()
c = parse_config()
if c.sign:
sys.stdout.flush()
c.key_id = c.sign
check_key_in_keys(c.key_id, c.local_keys)
import getpass
c.key_password = getpass.getpass('Enter GPG keystore password: ')
else:
c.key_id = None
c.key_password = None
if c.prepare:
rev = prepare(c.root, c.version, c.key_id, c.key_password)
else:
os.chdir(c.root)
rev = open('rev.txt', encoding='UTF-8').read()
if c.push_local:
url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local)
else:
url = None
if url is not None:
print(' URL: %s' % url)
print('Next run the smoker tester:')
p = re.compile(".*/")
m = p.match(sys.argv[0])
print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url))
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('Keyboard interrupt...exiting')
Reference in New Issue View Git Blame Copy Permalink