mirror of https://github.com/apache/lucene.git
LUCENE-8837 smokeTestRelease.py option --download-only (#702)
* LUCENE-8837: smokeTestRelease.py option --download-only Move download() and check_and() functions to scriptutil Add cwd param to run() function in scriptutil Move the check_ant function from buildAndPushRelease into scriptutil.py, and let it return the version.
This commit is contained in:
Jan Høydahl
GitHub
parent
88c5817c01
commit
44287d4206
|
|
@ -21,6 +21,7 @@ import os
|
|||
import sys
|
||||
import subprocess
|
||||
from subprocess import TimeoutExpired
|
||||
from scriptutil import check_ant
|
||||
import textwrap
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import xml.etree.ElementTree as ET
|
||||
|
|
@ -286,16 +287,6 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
|
|||
raise RuntimeError('"git --version" returned a non-zero exit code.')
|
||||
check_ant()
|
||||
|
||||
def check_ant():
|
||||
antVersion = os.popen('ant -version').read().strip()
|
||||
if (antVersion.startswith('Apache Ant(TM) version 1.8')):
|
||||
return
|
||||
if (antVersion.startswith('Apache Ant(TM) version 1.9')):
|
||||
return
|
||||
if (antVersion.startswith('Apache Ant(TM) version 1.10')):
|
||||
return
|
||||
raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
|
||||
|
||||
def check_key_in_keys(gpgKeyID, local_keys):
|
||||
if gpgKeyID is not None:
|
||||
print(' Verify your gpg key is in the main KEYS file')
|
||||
|
|
|
|||
|
|
@ -17,7 +17,11 @@ import argparse
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from enum import Enum
|
||||
import time
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import urllib.parse
|
||||
|
||||
class Version(object):
|
||||
def __init__(self, major, minor, bugfix, prerelease):
|
||||
|
|
@ -66,14 +70,19 @@ class Version(object):
|
|||
(self.bugfix > other.bugfix or self.bugfix == other.bugfix and
|
||||
self.prerelease >= other.prerelease)))
|
||||
|
||||
def gt(self, other):
|
||||
return (self.major > other.major or
|
||||
(self.major == other.major and self.minor > other.minor) or
|
||||
(self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix))
|
||||
|
||||
def is_back_compat_with(self, other):
|
||||
if not self.on_or_after(other):
|
||||
raise Exception('Back compat check disallowed for newer version: %s < %s' % (self, other))
|
||||
return other.major + 1 >= self.major
|
||||
|
||||
def run(cmd):
|
||||
def run(cmd, cwd=None):
|
||||
try:
|
||||
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
|
||||
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(e.output.decode('utf-8'))
|
||||
raise e
|
||||
|
|
@ -99,6 +108,18 @@ def update_file(filename, line_re, edit):
|
|||
f.write(''.join(buffer))
|
||||
return True
|
||||
|
||||
|
||||
def check_ant():
|
||||
antVersion = os.popen('ant -version').read().strip()
|
||||
if (antVersion.startswith('Apache Ant(TM) version 1.8')):
|
||||
return antVersion.split(" ")[3]
|
||||
if (antVersion.startswith('Apache Ant(TM) version 1.9')):
|
||||
return antVersion.split(" ")[3]
|
||||
if (antVersion.startswith('Apache Ant(TM) version 1.10')):
|
||||
return antVersion.split(" ")[3]
|
||||
raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion)
|
||||
|
||||
|
||||
# branch types are "release", "stable" and "unstable"
|
||||
class BranchType(Enum):
|
||||
unstable = 1
|
||||
|
|
@ -122,6 +143,49 @@ def find_branch_type():
|
|||
return BranchType.release
|
||||
raise Exception('Cannot run %s on feature branch' % sys.argv[0].rsplit('/', 1)[-1])
|
||||
|
||||
|
||||
def download(name, urlString, tmpDir, quiet=False, force_clean=True):
|
||||
if not quiet:
|
||||
print("Downloading %s" % urlString)
|
||||
startTime = time.time()
|
||||
fileName = '%s/%s' % (tmpDir, name)
|
||||
if not force_clean and os.path.exists(fileName):
|
||||
if not quiet and fileName.find('.asc') == -1:
|
||||
print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
|
||||
return
|
||||
try:
|
||||
attemptDownload(urlString, fileName)
|
||||
except Exception as e:
|
||||
print('Retrying download of url %s after exception: %s' % (urlString, e))
|
||||
try:
|
||||
attemptDownload(urlString, fileName)
|
||||
except Exception as e:
|
||||
raise RuntimeError('failed to download url "%s"' % urlString) from e
|
||||
if not quiet and fileName.find('.asc') == -1:
|
||||
t = time.time()-startTime
|
||||
sizeMB = os.path.getsize(fileName)/1024./1024.
|
||||
print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
|
||||
|
||||
|
||||
def attemptDownload(urlString, fileName):
|
||||
fIn = urllib.request.urlopen(urlString)
|
||||
fOut = open(fileName, 'wb')
|
||||
success = False
|
||||
try:
|
||||
while True:
|
||||
s = fIn.read(65536)
|
||||
if s == b'':
|
||||
break
|
||||
fOut.write(s)
|
||||
fOut.close()
|
||||
fIn.close()
|
||||
success = True
|
||||
finally:
|
||||
fIn.close()
|
||||
fOut.close()
|
||||
if not success:
|
||||
os.remove(fileName)
|
||||
|
||||
version_prop_re = re.compile('version\.base=(.*)')
|
||||
def find_current_version():
|
||||
return version_prop_re.search(open('lucene/version.properties').read()).group(1).strip()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
|
|
@ -14,35 +16,31 @@
|
|||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import zipfile
|
||||
import codecs
|
||||
import tarfile
|
||||
import zipfile
|
||||
import threading
|
||||
import traceback
|
||||
import datetime
|
||||
import time
|
||||
import subprocess
|
||||
import signal
|
||||
import shutil
|
||||
import filecmp
|
||||
import hashlib
|
||||
import http.client
|
||||
import re
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import urllib.parse
|
||||
import sys
|
||||
import html.parser
|
||||
from collections import defaultdict
|
||||
import xml.etree.ElementTree as ET
|
||||
import filecmp
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import traceback
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from collections import defaultdict
|
||||
from collections import namedtuple
|
||||
from scriptutil import download
|
||||
|
||||
import checkJavaDocs
|
||||
import checkJavadocLinks
|
||||
import io
|
||||
import codecs
|
||||
import textwrap
|
||||
from collections import namedtuple
|
||||
|
||||
# This tool expects to find /lucene and /solr off the base URL. You
|
||||
# must have a working gpg, tar, unzip in your path. This has been
|
||||
|
|
@ -111,44 +109,6 @@ def getHREFs(urlString):
|
|||
links.append((text, fullURL))
|
||||
return links
|
||||
|
||||
def download(name, urlString, tmpDir, quiet=False):
|
||||
startTime = time.time()
|
||||
fileName = '%s/%s' % (tmpDir, name)
|
||||
if not FORCE_CLEAN and os.path.exists(fileName):
|
||||
if not quiet and fileName.find('.asc') == -1:
|
||||
print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.))
|
||||
return
|
||||
try:
|
||||
attemptDownload(urlString, fileName)
|
||||
except Exception as e:
|
||||
print('Retrying download of url %s after exception: %s' % (urlString, e))
|
||||
try:
|
||||
attemptDownload(urlString, fileName)
|
||||
except Exception as e:
|
||||
raise RuntimeError('failed to download url "%s"' % urlString) from e
|
||||
if not quiet and fileName.find('.asc') == -1:
|
||||
t = time.time()-startTime
|
||||
sizeMB = os.path.getsize(fileName)/1024./1024.
|
||||
print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t))
|
||||
|
||||
def attemptDownload(urlString, fileName):
|
||||
fIn = urllib.request.urlopen(urlString)
|
||||
fOut = open(fileName, 'wb')
|
||||
success = False
|
||||
try:
|
||||
while True:
|
||||
s = fIn.read(65536)
|
||||
if s == b'':
|
||||
break
|
||||
fOut.write(s)
|
||||
fOut.close()
|
||||
fIn.close()
|
||||
success = True
|
||||
finally:
|
||||
fIn.close()
|
||||
fOut.close()
|
||||
if not success:
|
||||
os.remove(fileName)
|
||||
|
||||
def load(urlString):
|
||||
try:
|
||||
|
|
@ -362,13 +322,13 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
|
|||
|
||||
for artifact, urlString in artifacts:
|
||||
print(' download %s...' % artifact)
|
||||
download(artifact, urlString, tmpDir)
|
||||
download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
|
||||
verifyDigests(artifact, urlString, tmpDir)
|
||||
|
||||
if isSigned:
|
||||
print(' verify sig')
|
||||
# Test sig (this is done with a clean brand-new GPG world)
|
||||
download(artifact + '.asc', urlString + '.asc', tmpDir)
|
||||
download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN)
|
||||
sigFile = '%s/%s.asc' % (tmpDir, artifact)
|
||||
artifactFile = '%s/%s' % (tmpDir, artifact)
|
||||
logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact)
|
||||
|
|
@ -997,7 +957,7 @@ def getBinaryDistFiles(project, tmpDir, version, baseURL):
|
|||
if not os.path.exists('%s/%s' % (tmpDir, distribution)):
|
||||
distURL = '%s/%s/%s' % (baseURL, project, distribution)
|
||||
print(' download %s...' % distribution, end=' ')
|
||||
download(distribution, distURL, tmpDir)
|
||||
download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN)
|
||||
destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project)
|
||||
if os.path.exists(destDir):
|
||||
shutil.rmtree(destDir)
|
||||
|
|
@ -1226,7 +1186,7 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
|
|||
crawl(downloadedFiles, subURL, path, exclusions)
|
||||
else:
|
||||
if not os.path.exists(path) or FORCE_CLEAN:
|
||||
download(text, subURL, targetDir, quiet=True)
|
||||
download(text, subURL, targetDir, quiet=True, force_clean=FORCE_CLEAN)
|
||||
downloadedFiles.append(path)
|
||||
sys.stdout.write('.')
|
||||
|
||||
|
|
@ -1277,6 +1237,8 @@ def parse_config():
|
|||
help='Version of the release, defaults to that in URL')
|
||||
parser.add_argument('--test-java12', metavar='JAVA12_HOME',
|
||||
help='Path to Java12 home directory, to run tests with if specified')
|
||||
parser.add_argument('--download-only', action='store_true', default=False,
|
||||
help='Only perform download and sha hash check steps')
|
||||
parser.add_argument('url', help='Url pointing to release to test')
|
||||
parser.add_argument('test_args', nargs=argparse.REMAINDER,
|
||||
help='Arguments to pass to ant for testing, e.g. -Dwhat=ever.')
|
||||
|
|
@ -1445,10 +1407,10 @@ def main():
|
|||
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
|
||||
|
||||
print('NOTE: output encoding is %s' % sys.stdout.encoding)
|
||||
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args))
|
||||
|
||||
def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs):
|
||||
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args),
|
||||
downloadOnly=c.download_only)
|
||||
|
||||
def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
|
||||
startTime = datetime.datetime.now()
|
||||
|
||||
# disable flakey tests for smoke-tester runs:
|
||||
|
|
@ -1489,27 +1451,32 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
|
|||
else:
|
||||
keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
|
||||
print(" Downloading online KEYS file %s" % keysFileURL)
|
||||
download('KEYS', keysFileURL, tmpDir)
|
||||
download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN)
|
||||
keysFile = '%s/KEYS' % (tmpDir)
|
||||
|
||||
print()
|
||||
print('Test Lucene...')
|
||||
checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile)
|
||||
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
|
||||
unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
|
||||
unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
|
||||
if not downloadOnly:
|
||||
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
|
||||
unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
|
||||
unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
|
||||
else:
|
||||
print("\nLucene test done (--download-only specified)")
|
||||
|
||||
print()
|
||||
print('Test Solr...')
|
||||
checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile)
|
||||
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
|
||||
unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
|
||||
solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
|
||||
gitRevision, version, testArgs, baseURL)
|
||||
|
||||
print()
|
||||
print('Test Maven artifacts for Lucene and Solr...')
|
||||
checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
|
||||
if not downloadOnly:
|
||||
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
|
||||
unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
|
||||
solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
|
||||
gitRevision, version, testArgs, baseURL)
|
||||
print()
|
||||
print('Test Maven artifacts for Lucene and Solr...')
|
||||
checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
|
||||
else:
|
||||
print("Solr test done (--download-only specified)")
|
||||
|
||||
print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue