LUCENE-8837 smokeTestRelease.py option --download-only (#702)

* LUCENE-8837: smokeTestRelease.py option --download-only
Move download() and check_and() functions to scriptutil
Add cwd param to run() function in scriptutil
Move the check_ant function from buildAndPushRelease into scriptutil.py, and let it return the version.
This commit is contained in:
Jan Høydahl 2019-06-11 10:17:10 +02:00 committed by GitHub
parent 88c5817c01
commit 44287d4206
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 113 additions and 91 deletions

View File

@ -21,6 +21,7 @@ import os
import sys
import subprocess
from subprocess import TimeoutExpired
from scriptutil import check_ant
import textwrap
import urllib.request, urllib.error, urllib.parse
import xml.etree.ElementTree as ET
@ -286,16 +287,6 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
raise RuntimeError('"git --version" returned a non-zero exit code.')
check_ant()
def check_ant():
antVersion = os.popen('ant -version').read().strip()
if (antVersion.startswith('Apache Ant(TM) version 1.8')):
return
if (antVersion.startswith('Apache Ant(TM) version 1.9')):
return
if (antVersion.startswith('Apache Ant(TM) version 1.10')):
return
raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
def check_key_in_keys(gpgKeyID, local_keys):
if gpgKeyID is not None:
print(' Verify your gpg key is in the main KEYS file')

View File

@ -17,7 +17,11 @@ import argparse
import re
import subprocess
import sys
import os
from enum import Enum
import time
import urllib.request, urllib.error, urllib.parse
import urllib.parse
class Version(object):
def __init__(self, major, minor, bugfix, prerelease):
@ -66,14 +70,19 @@ class Version(object):
(self.bugfix > other.bugfix or self.bugfix == other.bugfix and
self.prerelease >= other.prerelease)))
def gt(self, other):
return (self.major > other.major or
(self.major == other.major and self.minor > other.minor) or
(self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix))
def is_back_compat_with(self, other):
if not self.on_or_after(other):
raise Exception('Back compat check disallowed for newer version: %s < %s' % (self, other))
return other.major + 1 >= self.major
def run(cmd):
def run(cmd, cwd=None):
try:
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
print(e.output.decode('utf-8'))
raise e
@ -99,6 +108,18 @@ def update_file(filename, line_re, edit):
f.write(''.join(buffer))
return True
def check_ant():
antVersion = os.popen('ant -version').read().strip()
if (antVersion.startswith('Apache Ant(TM) version 1.8')):
return antVersion.split(" ")[3]
if (antVersion.startswith('Apache Ant(TM) version 1.9')):
return antVersion.split(" ")[3]
if (antVersion.startswith('Apache Ant(TM) version 1.10')):
return antVersion.split(" ")[3]
raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
# branch types are "release", "stable" and "unstable"
class BranchType(Enum):
unstable = 1
@ -122,6 +143,49 @@ def find_branch_type():
return BranchType.release
raise Exception('Cannot run %s on feature branch' % sys.argv[0].rsplit('/', 1)[-1])
def download(name, urlString, tmpDir, quiet=False, force_clean=True):
if not quiet:
print("Downloading %s" % urlString)
startTime = time.time()
fileName = '%s/%s' % (tmpDir, name)
if not force_clean and os.path.exists(fileName):
if not quiet and fileName.find('.asc') == -1:
print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
return
try:
attemptDownload(urlString, fileName)
except Exception as e:
print('Retrying download of url %s after exception: %s' % (urlString, e))
try:
attemptDownload(urlString, fileName)
except Exception as e:
raise RuntimeError('failed to download url "%s"' % urlString) from e
if not quiet and fileName.find('.asc') == -1:
t = time.time()-startTime
sizeMB = os.path.getsize(fileName)/1024./1024.
print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
def attemptDownload(urlString, fileName):
fIn = urllib.request.urlopen(urlString)
fOut = open(fileName, 'wb')
success = False
try:
while True:
s = fIn.read(65536)
if s == b'':
break
fOut.write(s)
fOut.close()
fIn.close()
success = True
finally:
fIn.close()
fOut.close()
if not success:
os.remove(fileName)
version_prop_re = re.compile('version\.base=(.*)')
def find_current_version():
return version_prop_re.search(open('lucene/version.properties').read()).group(1).strip()

125
dev-tools/scripts/smokeTestRelease.py Normal file → Executable file
View File

@ -1,3 +1,5 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@ -14,35 +16,31 @@
# limitations under the License.
import argparse
import os
import zipfile
import codecs
import tarfile
import zipfile
import threading
import traceback
import datetime
import time
import subprocess
import signal
import shutil
import filecmp
import hashlib
import http.client
import re
import urllib.request, urllib.error, urllib.parse
import urllib.parse
import sys
import html.parser
from collections import defaultdict
import xml.etree.ElementTree as ET
import filecmp
import os
import platform
import re
import shutil
import subprocess
import sys
import textwrap
import traceback
import urllib.error
import urllib.parse
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
import zipfile
from collections import defaultdict
from collections import namedtuple
from scriptutil import download
import checkJavaDocs
import checkJavadocLinks
import io
import codecs
import textwrap
from collections import namedtuple
# This tool expects to find /lucene and /solr off the base URL. You
# must have a working gpg, tar, unzip in your path. This has been
@ -111,44 +109,6 @@ def getHREFs(urlString):
links.append((text, fullURL))
return links
def download(name, urlString, tmpDir, quiet=False):
startTime = time.time()
fileName = '%s/%s' % (tmpDir, name)
if not FORCE_CLEAN and os.path.exists(fileName):
if not quiet and fileName.find('.asc') == -1:
print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
return
try:
attemptDownload(urlString, fileName)
except Exception as e:
print('Retrying download of url %s after exception: %s' % (urlString, e))
try:
attemptDownload(urlString, fileName)
except Exception as e:
raise RuntimeError('failed to download url "%s"' % urlString) from e
if not quiet and fileName.find('.asc') == -1:
t = time.time()-startTime
sizeMB = os.path.getsize(fileName)/1024./1024.
print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
def attemptDownload(urlString, fileName):
fIn = urllib.request.urlopen(urlString)
fOut = open(fileName, 'wb')
success = False
try:
while True:
s = fIn.read(65536)
if s == b'':
break
fOut.write(s)
fOut.close()
fIn.close()
success = True
finally:
fIn.close()
fOut.close()
if not success:
os.remove(fileName)
def load(urlString):
try:
@ -362,13 +322,13 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
for artifact, urlString in artifacts:
print(' download %s...' % artifact)
download(artifact, urlString, tmpDir)
download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
verifyDigests(artifact, urlString, tmpDir)
if isSigned:
print(' verify sig')
# Test sig (this is done with a clean brand-new GPG world)
download(artifact + '.asc', urlString + '.asc', tmpDir)
download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN)
sigFile = '%s/%s.asc' % (tmpDir, artifact)
artifactFile = '%s/%s' % (tmpDir, artifact)
logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact)
@ -997,7 +957,7 @@ def getBinaryDistFiles(project, tmpDir, version, baseURL):
if not os.path.exists('%s/%s' % (tmpDir, distribution)):
distURL = '%s/%s/%s' % (baseURL, project, distribution)
print(' download %s...' % distribution, end=' ')
download(distribution, distURL, tmpDir)
download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN)
destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project)
if os.path.exists(destDir):
shutil.rmtree(destDir)
@ -1226,7 +1186,7 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
crawl(downloadedFiles, subURL, path, exclusions)
else:
if not os.path.exists(path) or FORCE_CLEAN:
download(text, subURL, targetDir, quiet=True)
download(text, subURL, targetDir, quiet=True, force_clean=FORCE_CLEAN)
downloadedFiles.append(path)
sys.stdout.write('.')
@ -1277,6 +1237,8 @@ def parse_config():
help='Version of the release, defaults to that in URL')
parser.add_argument('--test-java12', metavar='JAVA12_HOME',
help='Path to Java12 home directory, to run tests with if specified')
parser.add_argument('--download-only', action='store_true', default=False,
help='Only perform download and sha hash check steps')
parser.add_argument('url', help='Url pointing to release to test')
parser.add_argument('test_args', nargs=argparse.REMAINDER,
help='Arguments to pass to ant for testing, e.g. -Dwhat=ever.')
@ -1445,10 +1407,10 @@ def main():
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
print('NOTE: output encoding is %s' % sys.stdout.encoding)
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args))
def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs):
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args),
downloadOnly=c.download_only)
def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
startTime = datetime.datetime.now()
# disable flakey tests for smoke-tester runs:
@ -1489,27 +1451,32 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
else:
keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
print(" Downloading online KEYS file %s" % keysFileURL)
download('KEYS', keysFileURL, tmpDir)
download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN)
keysFile = '%s/KEYS' % (tmpDir)
print()
print('Test Lucene...')
checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile)
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
if not downloadOnly:
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
else:
print("\nLucene test done (--download-only specified)")
print()
print('Test Solr...')
checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile)
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
gitRevision, version, testArgs, baseURL)
print()
print('Test Maven artifacts for Lucene and Solr...')
checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
if not downloadOnly:
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
gitRevision, version, testArgs, baseURL)
print()
print('Test Maven artifacts for Lucene and Solr...')
checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
else:
print("Solr test done (--download-only specified)")
print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))