From 879e825083e6879fe76cbffe15de0d798c46f7f5 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Sun, 8 Apr 2012 18:22:14 +0000 Subject: [PATCH] LUCENE-3966: smoke tester can test from local (file://) urls, handle url-shortened base URLs, and build/stage/test unsigned artifacts git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1311067 13f79535-47bb-0310-9956-ffa450edef68 --- dev-tools/scripts/buildAndPushRelease.py | 163 +++++++++++++++++++---- dev-tools/scripts/smokeTestRelease.py | 122 +++++++++++------ lucene/build.xml | 3 +- solr/build.xml | 4 +- 4 files changed, 224 insertions(+), 68 deletions(-) diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py index 4e38b1e1722..8a60b365342 100644 --- a/dev-tools/scripts/buildAndPushRelease.py +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -19,17 +19,15 @@ import shutil import os import sys -# Usage: python -u buildRelease.py /path/to/checkout version(eg: 3.4.0) gpgKey(eg: 6E68DA61) rcNum user [-prepare] [-push] +# Usage: python -u buildRelease.py [-sign gpgKey(eg: 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName] [-smoke tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) # -# EG: python -u buildRelease.py -prepare -push /lucene/34x 3.4.0 6E68DA61 1 mikemccand -# +# EG: python -u buildRelease.py -prepare -push -sign 6E68DA61 mikemccand /lucene/34x 3.4.0 0 -# TODO: also run smokeTestRelease.py? - -# NOTE: you have to type in your gpg password at some point while this -# runs; it's VERY confusing because the output is directed to -# /tmp/release.log, so, you have to tail that and when GPG wants your -# password, type it! +# NOTE: if you specify -sign, you have to type in your gpg password at +# some point while this runs; it's VERY confusing because the output +# is directed to /tmp/release.log, so, you have to tail that and when +# GPG wants your password, type it! Also sometimes you have to type +# it twice in a row! LOG = '/tmp/release.log' @@ -70,7 +68,7 @@ def getSVNRev(): return rev -def prepare(root, version, gpgKeyID): +def prepare(root, version, gpgKeyID, doTest): print print 'Prepare release...' if os.path.exists(LOG): @@ -84,8 +82,10 @@ def prepare(root, version, gpgKeyID): print ' svn rev: %s' % rev log('\nSVN rev: %s\n' % rev) - print ' ant clean test' - run('ant clean test') + if doTest: + # Don't run tests if we are gonna smoke test after the release... + print ' ant clean test' + run('ant clean test') print ' clean checkout' scrubCheckout() @@ -93,10 +93,21 @@ def prepare(root, version, gpgKeyID): print ' lucene prepare-release' os.chdir('lucene') - run('ant -Dversion=%s -Dspecversion=%s -Dgpg.key=%s prepare-release' % (version, version, gpgKeyID)) + cmd = 'ant -Dversion=%s -Dspecversion=%s' % (version, version) + if gpgKeyID is not None: + cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID + else: + cmd += ' prepare-release-no-sign' + run(cmd) + print ' solr prepare-release' os.chdir('../solr') - run('ant -Dversion=%s -Dspecversion=%s -Dgpg.key=%s prepare-release' % (version, version, gpgKeyID)) + cmd = 'ant -Dversion=%s -Dspecversion=%s' % (version, version) + if gpgKeyID is not None: + cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID + else: + cmd += ' prepare-release-no-sign' + run(cmd) print ' done!' print return rev @@ -149,30 +160,134 @@ def push(version, root, rev, rcNum, username): print ' chmod...' run('ssh %s@people.apache.org "chmod -R a+rX-w public_html/staging_area/%s"' % (username, dir)) - print ' done! URL: https://people.apache.org/~%s/staging_area/%s' % (username, dir) + print ' done!' + url = 'https://people.apache.org/~%s/staging_area/%s' % (username, dir) + return url +def pushLocal(version, root, rev, rcNum, localDir): + print 'Push local [%s]...' % localDir + os.makedirs(localDir) + + dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev) + os.makedirs('%s/%s/lucene' % (localDir, dir)) + os.makedirs('%s/%s/solr' % (localDir, dir)) + print ' Lucene' + os.chdir('%s/lucene/dist' % root) + print ' zip...' + if os.path.exists('lucene.tar.bz2'): + os.remove('lucene.tar.bz2') + run('tar cjf lucene.tar.bz2 *') + + os.chdir('%s/%s/lucene' % (localDir, dir)) + print ' unzip...' + run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root) + os.remove('%s/lucene/dist/lucene.tar.bz2' % root) + print ' copy changes...' + run('cp -r "%s/lucene/build/docs/changes" changes-%s' % (root, version)) + + print ' Solr' + os.chdir('%s/solr/package' % root) + print ' zip...' + if os.path.exists('solr.tar.bz2'): + os.remove('solr.tar.bz2') + run('tar cjf solr.tar.bz2 *') + print ' unzip...' + os.chdir('%s/%s/solr' % (localDir, dir)) + run('tar xjf "%s/solr/package/solr.tar.bz2"' % root) + os.remove('%s/solr/package/solr.tar.bz2' % root) + + print ' KEYS' + run('wget http://people.apache.org/keys/group/lucene.asc') + os.rename('lucene.asc', 'KEYS') + run('chmod a+r-w KEYS') + run('cp KEYS ../lucene') + + print ' chmod...' + os.chdir('..') + run('chmod -R a+rX-w .') + + print ' done!' + return 'file://%s/%s' % (os.path.abspath(localDir), dir) def main(): doPrepare = '-prepare' in sys.argv if doPrepare: sys.argv.remove('-prepare') - doPush = '-push' in sys.argv - if doPush: - sys.argv.remove('-push') + + try: + idx = sys.argv.index('-push') + except ValueError: + doPushRemote = False + else: + doPushRemote = True + username = sys.argv[idx+1] + del sys.argv[idx:idx+2] + + try: + idx = sys.argv.index('-smoke') + except ValueError: + smokeTmpDir = None + else: + smokeTmpDir = sys.argv[idx+1] + del sys.argv[idx:idx+2] + if os.path.exists(smokeTmpDir): + print + print 'ERROR: smoke tmpDir "%s" exists; please remove first' % smokeTmpDir + print + sys.exit(1) + + try: + idx = sys.argv.index('-pushLocal') + except ValueError: + doPushLocal = False + else: + doPushLocal = True + localStagingDir = sys.argv[idx+1] + del sys.argv[idx:idx+2] + if os.path.exists(localStagingDir): + print + print 'ERROR: pushLocal dir "%s" exists; please remove first' % localStagingDir + print + sys.exit(1) + + if doPushRemote and doPushLocal: + print + print 'ERROR: specify at most one of -push or -pushLocal (got both)' + print + sys.exit(1) + + try: + idx = sys.argv.index('-sign') + except ValueError: + gpgKeyID = None + else: + gpgKeyID = sys.argv[idx+1] + del sys.argv[idx:idx+2] + root = os.path.abspath(sys.argv[1]) version = sys.argv[2] - gpgKeyID = sys.argv[3] - rcNum = int(sys.argv[4]) - username = sys.argv[5] + rcNum = int(sys.argv[3]) if doPrepare: - rev = prepare(root, version, gpgKeyID) + rev = prepare(root, version, gpgKeyID, smokeTmpDir is None) else: os.chdir(root) rev = open('rev.txt').read() - if doPush: - push(version, root, rev, rcNum, username) + if doPushRemote: + url = push(version, root, rev, rcNum, username) + elif doPushLocal: + url = pushLocal(version, root, rev, rcNum, localStagingDir) + else: + url = NOne + + if url is not None: + print ' URL: %s' % url + + if smokeTmpDir is not None: + import smokeTestRelease + smokeTestRelease.DEBUG = False + smokeTestRelease.smokeTest(url, version, smokeTmpDir, gpgKeyID is not None) if __name__ == '__main__': main() diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index b4452c6fd10..cfc14fd2206 100644 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -36,6 +36,16 @@ import checkJavaDocs # must have a working gpg, tar, unzip in your path. This has been # tested on Linux and on Cygwin under Windows 7. +def unshortenURL(url): + parsed = urlparse.urlparse(url) + if parsed[0] in ('http', 'https'): + h = httplib.HTTPConnection(parsed.netloc) + h.request('HEAD', parsed.path) + response = h.getresponse() + if response.status/100 == 3 and response.getheader('Location'): + return response.getheader('Location') + return url + def javaExe(version): if version == '1.5': path = JAVA5_HOME @@ -143,7 +153,7 @@ def download(name, urlString, tmpDir, quiet=False): def load(urlString): return urllib2.urlopen(urlString).read() -def checkSigs(project, urlString, version, tmpDir): +def checkSigs(project, urlString, version, tmpDir, isSigned): print ' test basics...' ents = getDirEntries(urlString) @@ -151,7 +161,11 @@ def checkSigs(project, urlString, version, tmpDir): keysURL = None changesURL = None mavenURL = None - expectedSigs = ['asc', 'md5', 'sha1'] + expectedSigs = [] + if isSigned: + expectedSigs.append('asc') + expectedSigs.extend(['md5', 'sha1']) + artifacts = [] for text, subURL in ents: if text == 'KEYS': @@ -239,34 +253,35 @@ def checkSigs(project, urlString, version, tmpDir): download(artifact, urlString, tmpDir) verifyDigests(artifact, urlString, tmpDir) - print ' verify sig' - # Test sig (this is done with a clean brand-new GPG world) - download(artifact + '.asc', urlString + '.asc', tmpDir) - sigFile = '%s/%s.asc' % (tmpDir, artifact) - artifactFile = '%s/%s' % (tmpDir, artifact) - logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact) - run('gpg --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile), - logFile) - # Forward any GPG warnings, except the expected one (since its a clean world) - f = open(logFile, 'rb') - for line in f.readlines(): - if line.lower().find('warning') != -1 \ - and line.find('WARNING: This key is not certified with a trusted signature') == -1: - print ' GPG: %s' % line.strip() - f.close() + if isSigned: + print ' verify sig' + # Test sig (this is done with a clean brand-new GPG world) + download(artifact + '.asc', urlString + '.asc', tmpDir) + sigFile = '%s/%s.asc' % (tmpDir, artifact) + artifactFile = '%s/%s' % (tmpDir, artifact) + logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact) + run('gpg --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile), + logFile) + # Forward any GPG warnings, except the expected one (since its a clean world) + f = open(logFile, 'rb') + for line in f.readlines(): + if line.lower().find('warning') != -1 \ + and line.find('WARNING: This key is not certified with a trusted signature') == -1: + print ' GPG: %s' % line.strip() + f.close() - # Test trust (this is done with the real users config) - run('gpg --import %s' % (keysFile), - '%s/%s.gpg.trust.import.log 2>&1' % (tmpDir, project)) - print ' verify trust' - logFile = '%s/%s.%s.gpg.trust.log' % (tmpDir, project, artifact) - run('gpg --verify %s %s' % (sigFile, artifactFile), logFile) - # Forward any GPG warnings: - f = open(logFile, 'rb') - for line in f.readlines(): - if line.lower().find('warning') != -1: - print ' GPG: %s' % line.strip() - f.close() + # Test trust (this is done with the real users config) + run('gpg --import %s' % (keysFile), + '%s/%s.gpg.trust.import.log 2>&1' % (tmpDir, project)) + print ' verify trust' + logFile = '%s/%s.%s.gpg.trust.log' % (tmpDir, project, artifact) + run('gpg --verify %s %s' % (sigFile, artifactFile), logFile) + # Forward any GPG warnings: + f = open(logFile, 'rb') + for line in f.readlines(): + if line.lower().find('warning') != -1: + print ' GPG: %s' % line.strip() + f.close() def testChanges(project, version, changesURLString): print ' check changes HTML...' @@ -370,10 +385,24 @@ def verifyDigests(artifact, urlString, tmpDir): raise RuntimeError('SHA1 digest mismatch for %s: expected %s but got %s' % (artifact, sha1Expected, sha1Actual)) def getDirEntries(urlString): - links = getHREFs(urlString) - for i, (text, subURL) in enumerate(links): - if text == 'Parent Directory' or text == '..': - return links[(i+1):] + if urlString.startswith('file://'): + path = urlString[7:] + if path.endswith('/'): + path = path[:-1] + l = [] + for ent in os.listdir(path): + entPath = '%s/%s' % (path, ent) + if os.path.isdir(entPath): + entPath += '/' + ent += '/' + l.append((ent, 'file://%s' % entPath)) + l.sort() + return l + else: + links = getHREFs(urlString) + for i, (text, subURL) in enumerate(links): + if text == 'Parent Directory' or text == '..': + return links[(i+1):] def unpack(project, tmpDir, artifact, version): destDir = '%s/unpack' % tmpDir @@ -642,7 +671,7 @@ def testDemo(isSrc, version): raise RuntimeError('lucene demo\'s SearchFiles found too few results: %s' % numHits) print ' got %d hits for query "lucene"' % numHits -def checkMaven(baseURL, tmpDir, version): +def checkMaven(baseURL, tmpDir, version, isSigned): # Locate the release branch in subversion m = re.match('(\d+)\.(\d+)', version) # Get Major.minor version components releaseBranchText = 'lucene_solr_%s_%s/' % (m.group(1), m.group(2)) @@ -680,8 +709,9 @@ def checkMaven(baseURL, tmpDir, version): checkJavadocAndSourceArtifacts(nonMavenizedDeps, artifacts, version) print " verify deployed POMs' coordinates..." verifyDeployedPOMsCoordinates(artifacts, version) - print ' verify maven artifact sigs', - verifyMavenSigs(baseURL, tmpDir, artifacts) + if isSigned: + print ' verify maven artifact sigs', + verifyMavenSigs(baseURL, tmpDir, artifacts) distributionFiles = getDistributionsForMavenChecks(tmpDir, version, baseURL) @@ -1004,7 +1034,7 @@ def getPOMtemplates(POMtemplates, tmpDir, releaseBranchSvnURL): if POMtemplates['solr'] is None: raise RuntimeError('No Solr POMs found at %s' % sourceLocation) POMtemplates['grandfather'] = [p for p in allPOMtemplates if '/maven/pom.xml.template' in p] - if POMtemplates['grandfather'] is None: + if len(POMtemplates['grandfather']) == 0: raise RuntimeError('No Lucene/Solr grandfather POM found at %s' % sourceLocation) def crawl(downloadedFiles, urlString, targetDir, exclusions=set()): @@ -1033,6 +1063,10 @@ def main(): version = sys.argv[2] tmpDir = os.path.abspath(sys.argv[3]) + smokeTest(baseURL, version, tmpDir, True) + +def smokeTest(baseURL, version, tmpDir, isSigned): + if not DEBUG: if os.path.exists(tmpDir): raise RuntimeError('temp dir %s exists; please remove first' % tmpDir) @@ -1042,7 +1076,13 @@ def main(): lucenePath = None solrPath = None - print 'Load release URL...' + print + print 'Load release URL "%s"...' % baseURL + newBaseURL = unshortenURL(baseURL) + if newBaseURL != baseURL: + print ' unshortened: %s' % newBaseURL + baseURL = newBaseURL + for text, subURL in getDirEntries(baseURL): if text.lower().find('lucene') != -1: lucenePath = subURL @@ -1056,20 +1096,20 @@ def main(): print print 'Test Lucene...' - checkSigs('lucene', lucenePath, version, tmpDir) + checkSigs('lucene', lucenePath, version, tmpDir, isSigned) for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version): unpack('lucene', tmpDir, artifact, version) unpack('lucene', tmpDir, 'lucene-%s-src.tgz' % version, version) print print 'Test Solr...' - checkSigs('solr', solrPath, version, tmpDir) + checkSigs('solr', solrPath, version, tmpDir, isSigned) for artifact in ('apache-solr-%s.tgz' % version, 'apache-solr-%s.zip' % version): unpack('solr', tmpDir, artifact, version) unpack('solr', tmpDir, 'apache-solr-%s-src.tgz' % version, version) print 'Test Maven artifacts for Lucene and Solr...' - checkMaven(baseURL, tmpDir, version) + checkMaven(baseURL, tmpDir, version, isSigned) if __name__ == '__main__': main() diff --git a/lucene/build.xml b/lucene/build.xml index 03ddbc54553..2d3e512f87b 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -452,7 +452,8 @@ - + + diff --git a/solr/build.xml b/solr/build.xml index c95d4836177..b42baf43ac7 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -330,8 +330,8 @@ - + +