#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import datetime import re import time import os import sys import subprocess from subprocess import TimeoutExpired from scriptutil import check_ant import textwrap import urllib.request, urllib.error, urllib.parse import xml.etree.ElementTree as ET LOG = '/tmp/release.log' def log(msg): f = open(LOG, mode='ab') f.write(msg.encode('utf-8')) f.close() def run(command): log('\n\n%s: RUN: %s\n' % (datetime.datetime.now(), command)) if os.system('%s >> %s 2>&1' % (command, LOG)): msg = ' FAILED: %s [see log %s]' % (command, LOG) print(msg) raise RuntimeError(msg) def runAndSendGPGPassword(command, password): p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE) f = open(LOG, 'ab') while True: p.stdout.flush() line = p.stdout.readline() if len(line) == 0: break f.write(line) if line.find(b'Enter GPG keystore password:') != -1: time.sleep(1.0) p.stdin.write((password + '\n').encode('UTF-8')) p.stdin.write('\n'.encode('UTF-8')) try: result = p.wait(timeout=120) if result != 0: msg = ' FAILED: %s [see log %s]' % (command, LOG) print(msg) raise RuntimeError(msg) except TimeoutExpired: msg = ' FAILED: %s [timed out after 2 minutes; see log %s]' % (command, LOG) print(msg) raise RuntimeError(msg) def load(urlString, encoding="utf-8"): try: content = urllib.request.urlopen(urlString).read().decode(encoding) except Exception as e: print('Retrying download of url %s after exception: %s' % (urlString, e)) content = urllib.request.urlopen(urlString).read().decode(encoding) return content def getGitRev(): status = os.popen('git status').read().strip() if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status: raise RuntimeError('git clone is dirty:\n\n%s' % status) branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip() command = 'git log origin/%s..' % branch p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() if len(stdout.strip()) > 0: raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, stdout.decode('utf-8'))) if len(stderr.strip()) > 0: raise RuntimeError('Command "%s" failed:\n\n%s' % (command, stderr.decode('utf-8'))) print(' git clone is clean') return os.popen('git rev-parse HEAD').read().strip() def prepare(root, version, gpgKeyID, gpgPassword): print() print('Prepare release...') if os.path.exists(LOG): os.remove(LOG) os.chdir(root) print(' git pull...') run('git pull') rev = getGitRev() print(' git rev: %s' % rev) log('\nGIT rev: %s\n' % rev) print(' Check DOAP files') checkDOAPfiles(version) print(' ant -Dtests.badapples=false clean validate documentation-lint test') run('ant -Dtests.badapples=false clean validate documentation-lint test') open('rev.txt', mode='wb').write(rev.encode('UTF-8')) print(' lucene prepare-release') os.chdir('lucene') cmd = 'ant -Dversion=%s' % version if gpgKeyID is not None: cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID else: cmd += ' prepare-release-no-sign' if gpgPassword is not None: runAndSendGPGPassword(cmd, gpgPassword) else: run(cmd) print(' solr prepare-release') os.chdir('../solr') cmd = 'ant -Dversion=%s' % version if gpgKeyID is not None: cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID else: cmd += ' prepare-release-no-sign' if gpgPassword is not None: runAndSendGPGPassword(cmd, gpgPassword) else: run(cmd) print(' done!') print() return rev reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE) reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE) reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE) def checkDOAPfiles(version): # In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced. errorMessages = [] for product in 'lucene', 'solr': url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product) distpage = load(url) releases = set() for regex in reVersion1, reVersion2: for tup in regex.findall(distpage): if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases continue releases.add(normalizeVersion(tup)) doapNS = '{http://usefulinc.com/ns/doap#}' xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS) doapFile = "dev-tools/doap/%s.rdf" % product treeRoot = ET.parse(doapFile).getroot() doapRevisions = set() for revision in treeRoot.findall(xpathRevision): match = reDoapRevision.match(revision.text) if (match is not None): if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions doapRevisions.add(normalizeVersion(match.groups())) else: errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile)) missingDoapRevisions = set() for release in releases: if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced missingDoapRevisions.add(release) if len(missingDoapRevisions) > 0: errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions)))) if (len(errorMessages) > 0): raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)' % '\n'.join(errorMessages)) def normalizeVersion(tup): suffix = '' if tup[-1] is not None and tup[-1].lower() == '-alpha': tup = tup[:(len(tup) - 1)] suffix = '-ALPHA' elif tup[-1] is not None and tup[-1].lower() == '-beta': tup = tup[:(len(tup) - 1)] suffix = '-BETA' while tup[-1] in ('', None): tup = tup[:(len(tup) - 1)] while len(tup) < 3: tup = tup + ('0',) return '.'.join(tup) + suffix def pushLocal(version, root, rev, rcNum, localDir): print('Push local [%s]...' % localDir) os.makedirs(localDir) dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev) os.makedirs('%s/%s/lucene' % (localDir, dir)) os.makedirs('%s/%s/solr' % (localDir, dir)) print(' Lucene') os.chdir('%s/lucene/dist' % root) print(' zip...') if os.path.exists('lucene.tar.bz2'): os.remove('lucene.tar.bz2') run('tar cjf lucene.tar.bz2 *') os.chdir('%s/%s/lucene' % (localDir, dir)) print(' unzip...') run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root) os.remove('%s/lucene/dist/lucene.tar.bz2' % root) print(' Solr') os.chdir('%s/solr/package' % root) print(' zip...') if os.path.exists('solr.tar.bz2'): os.remove('solr.tar.bz2') run('tar cjf solr.tar.bz2 *') print(' unzip...') os.chdir('%s/%s/solr' % (localDir, dir)) run('tar xjf "%s/solr/package/solr.tar.bz2"' % root) os.remove('%s/solr/package/solr.tar.bz2' % root) print(' chmod...') os.chdir('..') run('chmod -R a+rX-w .') print(' done!') return 'file://%s/%s' % (os.path.abspath(localDir), dir) def read_version(path): version_props_file = os.path.join(path, 'lucene', 'version.properties') return re.search(r'version\.base=(.*)', open(version_props_file).read()).group(1) def parse_config(): epilogue = textwrap.dedent(''' Example usage for a Release Manager: python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1 ''') description = 'Utility to build, push, and test a release.' parser = argparse.ArgumentParser(description=description, epilog=epilogue, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--no-prepare', dest='prepare', default=True, action='store_false', help='Use the already built release in the provided checkout') parser.add_argument('--local-keys', metavar='PATH', help='Uses local KEYS file to validate presence of RM\'s gpg key') parser.add_argument('--push-local', metavar='PATH', help='Push the release to the local path') parser.add_argument('--sign', metavar='KEYID', help='Sign the release with the given gpg key') parser.add_argument('--rc-num', metavar='NUM', type=int, default=1, help='Release Candidate number. Default: 1') parser.add_argument('--root', metavar='PATH', default='.', help='Root of Git working tree for lucene-solr. Default: "." (the current directory)') parser.add_argument('--logfile', metavar='PATH', help='Specify log file path (default /tmp/release.log)') config = parser.parse_args() if not config.prepare and config.sign: parser.error('Cannot sign already built release') if config.push_local is not None and os.path.exists(config.push_local): parser.error('Cannot push to local path that already exists') if config.rc_num <= 0: parser.error('Release Candidate number must be a positive integer') if not os.path.isdir(config.root): parser.error('Root path "%s" is not a directory' % config.root) if config.local_keys is not None and not os.path.exists(config.local_keys): parser.error('Local KEYS file "%s" not found' % config.local_keys) cwd = os.getcwd() os.chdir(config.root) config.root = os.getcwd() # Absolutize root dir if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]): parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root) os.chdir(cwd) global LOG if config.logfile: LOG = config.logfile config.version = read_version(config.root) print('Building version: %s' % config.version) return config def check_cmdline_tools(): # Fail fast if there are cmdline tool problems if os.system('git --version >/dev/null 2>/dev/null'): raise RuntimeError('"git --version" returned a non-zero exit code.') check_ant() def check_key_in_keys(gpgKeyID, local_keys): if gpgKeyID is not None: print(' Verify your gpg key is in the main KEYS file') if local_keys is not None: print(" Using local KEYS file %s" % local_keys) keysFileText = open(local_keys, encoding='iso-8859-1').read() keysFileLocation = local_keys else: keysFileURL = "https://archive.apache.org/dist/lucene/KEYS" keysFileLocation = keysFileURL print(" Using online KEYS file %s" % keysFileURL) keysFileText = load(keysFileURL, encoding='iso-8859-1') if len(gpgKeyID) > 2 and gpgKeyID[0:2] == '0x': gpgKeyID = gpgKeyID[2:] if len(gpgKeyID) > 40: gpgKeyID = gpgKeyID.replace(" ", "") if len(gpgKeyID) == 8: gpgKeyID8Char = "%s %s" % (gpgKeyID[0:4], gpgKeyID[4:8]) re_to_match = r"^pub .*\n\s+(\w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} %s|\w{32}%s)" % (gpgKeyID8Char, gpgKeyID) elif len(gpgKeyID) == 40: gpgKeyID40Char = "%s %s %s %s %s %s %s %s %s %s" % \ (gpgKeyID[0:4], gpgKeyID[4:8], gpgKeyID[8:12], gpgKeyID[12:16], gpgKeyID[16:20], gpgKeyID[20:24], gpgKeyID[24:28], gpgKeyID[28:32], gpgKeyID[32:36], gpgKeyID[36:]) re_to_match = r"^pub .*\n\s+(%s|%s)" % (gpgKeyID40Char, gpgKeyID) else: print('Invalid gpg key id format [%s]. Must be 8 byte short ID or 40 byte fingerprint, with or without 0x prefix, no spaces.' % gpgKeyID) exit(2) if re.search(re_to_match, keysFileText, re.MULTILINE): print(' Found key %s in KEYS file at %s' % (gpgKeyID, keysFileLocation)) else: print(' ERROR: Did not find your key %s in KEYS file at %s. Please add it and try again.' % (gpgKeyID, keysFileLocation)) if local_keys is not None: print(' You are using a local KEYS file. Make sure it is up to date or validate against the online version') exit(2) def main(): check_cmdline_tools() c = parse_config() if c.sign: sys.stdout.flush() c.key_id = c.sign check_key_in_keys(c.key_id, c.local_keys) import getpass c.key_password = getpass.getpass('Enter GPG keystore password: ') else: c.key_id = None c.key_password = None if c.prepare: rev = prepare(c.root, c.version, c.key_id, c.key_password) else: os.chdir(c.root) rev = open('rev.txt', encoding='UTF-8').read() if c.push_local: url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local) else: url = None if url is not None: print(' URL: %s' % url) print('Next run the smoker tester:') p = re.compile(".*/") m = p.match(sys.argv[0]) print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url)) if __name__ == '__main__': try: main() except KeyboardInterrupt: print('Keyboard interrupt...exiting')