From 9f29c0cc3f41027c3b553636dc0f36f367339adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 11 Jun 2019 10:17:10 +0200 Subject: [PATCH] LUCENE-8837 smokeTestRelease.py option --download-only (#702) * LUCENE-8837: smokeTestRelease.py option --download-only Move download() and check_and() functions to scriptutil Add cwd param to run() function in scriptutil Move the check_ant function from buildAndPushRelease into scriptutil.py, and let it return the version. (cherry picked from commit 44287d420624fe669a319f95f4712486bd57ae38) --- dev-tools/scripts/buildAndPushRelease.py | 11 +- dev-tools/scripts/scriptutil.py | 68 +++++++++++- dev-tools/scripts/smokeTestRelease.py | 125 +++++++++-------------- 3 files changed, 113 insertions(+), 91 deletions(-) mode change 100644 => 100755 dev-tools/scripts/smokeTestRelease.py diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py index 66fbd3fc190..0667be995c6 100644 --- a/dev-tools/scripts/buildAndPushRelease.py +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -21,6 +21,7 @@ import os import sys import subprocess from subprocess import TimeoutExpired +from scriptutil import check_ant import textwrap import urllib.request, urllib.error, urllib.parse import xml.etree.ElementTree as ET @@ -286,16 +287,6 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems raise RuntimeError('"git --version" returned a non-zero exit code.') check_ant() -def check_ant(): - antVersion = os.popen('ant -version').read().strip() - if (antVersion.startswith('Apache Ant(TM) version 1.8')): - return - if (antVersion.startswith('Apache Ant(TM) version 1.9')): - return - if (antVersion.startswith('Apache Ant(TM) version 1.10')): - return - raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion) - def check_key_in_keys(gpgKeyID, local_keys): if gpgKeyID is not None: print(' Verify your gpg key is in the main KEYS file') diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py index 9b264138349..6a30508d669 100644 --- a/dev-tools/scripts/scriptutil.py +++ b/dev-tools/scripts/scriptutil.py @@ -17,7 +17,11 @@ import argparse import re import subprocess import sys +import os from enum import Enum +import time +import urllib.request, urllib.error, urllib.parse +import urllib.parse class Version(object): def __init__(self, major, minor, bugfix, prerelease): @@ -66,14 +70,19 @@ class Version(object): (self.bugfix > other.bugfix or self.bugfix == other.bugfix and self.prerelease >= other.prerelease))) + def gt(self, other): + return (self.major > other.major or + (self.major == other.major and self.minor > other.minor) or + (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix)) + def is_back_compat_with(self, other): if not self.on_or_after(other): raise Exception('Back compat check disallowed for newer version: %s < %s' % (self, other)) return other.major + 1 >= self.major -def run(cmd): +def run(cmd, cwd=None): try: - output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd) except subprocess.CalledProcessError as e: print(e.output.decode('utf-8')) raise e @@ -99,6 +108,18 @@ def update_file(filename, line_re, edit): f.write(''.join(buffer)) return True + +def check_ant(): + antVersion = os.popen('ant -version').read().strip() + if (antVersion.startswith('Apache Ant(TM) version 1.8')): + return antVersion.split(" ")[3] + if (antVersion.startswith('Apache Ant(TM) version 1.9')): + return antVersion.split(" ")[3] + if (antVersion.startswith('Apache Ant(TM) version 1.10')): + return antVersion.split(" ")[3] + raise RuntimeError('Unsupported ant version (must be 1.8 - 1.10): "%s"' % antVersion) + + # branch types are "release", "stable" and "unstable" class BranchType(Enum): unstable = 1 @@ -122,6 +143,49 @@ def find_branch_type(): return BranchType.release raise Exception('Cannot run %s on feature branch' % sys.argv[0].rsplit('/', 1)[-1]) + +def download(name, urlString, tmpDir, quiet=False, force_clean=True): + if not quiet: + print("Downloading %s" % urlString) + startTime = time.time() + fileName = '%s/%s' % (tmpDir, name) + if not force_clean and os.path.exists(fileName): + if not quiet and fileName.find('.asc') == -1: + print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.)) + return + try: + attemptDownload(urlString, fileName) + except Exception as e: + print('Retrying download of url %s after exception: %s' % (urlString, e)) + try: + attemptDownload(urlString, fileName) + except Exception as e: + raise RuntimeError('failed to download url "%s"' % urlString) from e + if not quiet and fileName.find('.asc') == -1: + t = time.time()-startTime + sizeMB = os.path.getsize(fileName)/1024./1024. + print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t)) + + +def attemptDownload(urlString, fileName): + fIn = urllib.request.urlopen(urlString) + fOut = open(fileName, 'wb') + success = False + try: + while True: + s = fIn.read(65536) + if s == b'': + break + fOut.write(s) + fOut.close() + fIn.close() + success = True + finally: + fIn.close() + fOut.close() + if not success: + os.remove(fileName) + version_prop_re = re.compile('version\.base=(.*)') def find_current_version(): return version_prop_re.search(open('lucene/version.properties').read()).group(1).strip() diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py old mode 100644 new mode 100755 index 614a9416ca5..8aa237bacc2 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,35 +16,31 @@ # limitations under the License. import argparse -import os -import zipfile import codecs -import tarfile -import zipfile -import threading -import traceback import datetime -import time -import subprocess -import signal -import shutil +import filecmp import hashlib import http.client -import re -import urllib.request, urllib.error, urllib.parse -import urllib.parse -import sys -import html.parser -from collections import defaultdict -import xml.etree.ElementTree as ET -import filecmp +import os import platform +import re +import shutil +import subprocess +import sys +import textwrap +import traceback +import urllib.error +import urllib.parse +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +import zipfile +from collections import defaultdict +from collections import namedtuple +from scriptutil import download + import checkJavaDocs import checkJavadocLinks -import io -import codecs -import textwrap -from collections import namedtuple # This tool expects to find /lucene and /solr off the base URL. You # must have a working gpg, tar, unzip in your path. This has been @@ -111,44 +109,6 @@ def getHREFs(urlString): links.append((text, fullURL)) return links -def download(name, urlString, tmpDir, quiet=False): - startTime = time.time() - fileName = '%s/%s' % (tmpDir, name) - if not FORCE_CLEAN and os.path.exists(fileName): - if not quiet and fileName.find('.asc') == -1: - print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.)) - return - try: - attemptDownload(urlString, fileName) - except Exception as e: - print('Retrying download of url %s after exception: %s' % (urlString, e)) - try: - attemptDownload(urlString, fileName) - except Exception as e: - raise RuntimeError('failed to download url "%s"' % urlString) from e - if not quiet and fileName.find('.asc') == -1: - t = time.time()-startTime - sizeMB = os.path.getsize(fileName)/1024./1024. - print(' %.1f MB in %.2f sec (%.1f MB/sec)' % (sizeMB, t, sizeMB/t)) - -def attemptDownload(urlString, fileName): - fIn = urllib.request.urlopen(urlString) - fOut = open(fileName, 'wb') - success = False - try: - while True: - s = fIn.read(65536) - if s == b'': - break - fOut.write(s) - fOut.close() - fIn.close() - success = True - finally: - fIn.close() - fOut.close() - if not success: - os.remove(fileName) def load(urlString): try: @@ -362,13 +322,13 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile): for artifact, urlString in artifacts: print(' download %s...' % artifact) - download(artifact, urlString, tmpDir) + download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN) verifyDigests(artifact, urlString, tmpDir) if isSigned: print(' verify sig') # Test sig (this is done with a clean brand-new GPG world) - download(artifact + '.asc', urlString + '.asc', tmpDir) + download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN) sigFile = '%s/%s.asc' % (tmpDir, artifact) artifactFile = '%s/%s' % (tmpDir, artifact) logFile = '%s/%s.%s.gpg.verify.log' % (tmpDir, project, artifact) @@ -997,7 +957,7 @@ def getBinaryDistFiles(project, tmpDir, version, baseURL): if not os.path.exists('%s/%s' % (tmpDir, distribution)): distURL = '%s/%s/%s' % (baseURL, project, distribution) print(' download %s...' % distribution, end=' ') - download(distribution, distURL, tmpDir) + download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN) destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project) if os.path.exists(destDir): shutil.rmtree(destDir) @@ -1226,7 +1186,7 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()): crawl(downloadedFiles, subURL, path, exclusions) else: if not os.path.exists(path) or FORCE_CLEAN: - download(text, subURL, targetDir, quiet=True) + download(text, subURL, targetDir, quiet=True, force_clean=FORCE_CLEAN) downloadedFiles.append(path) sys.stdout.write('.') @@ -1277,6 +1237,8 @@ def parse_config(): help='Version of the release, defaults to that in URL') parser.add_argument('--test-java9', metavar='JAVA9_HOME', help='Path to Java9 home directory, to run tests with if specified') + parser.add_argument('--download-only', action='store_true', default=False, + help='Only perform download and sha hash check steps') parser.add_argument('url', help='Url pointing to release to test') parser.add_argument('test_args', nargs=argparse.REMAINDER, help='Arguments to pass to ant for testing, e.g. -Dwhat=ever.') @@ -1445,10 +1407,10 @@ def main(): raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version)) print('NOTE: output encoding is %s' % sys.stdout.encoding) - smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args)) - -def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs): + smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args), + downloadOnly=c.download_only) +def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False): startTime = datetime.datetime.now() # disable flakey tests for smoke-tester runs: @@ -1489,27 +1451,32 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, else: keysFileURL = "https://archive.apache.org/dist/lucene/KEYS" print(" Downloading online KEYS file %s" % keysFileURL) - download('KEYS', keysFileURL, tmpDir) + download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN) keysFile = '%s/KEYS' % (tmpDir) print() print('Test Lucene...') checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile) - for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version): - unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL) - unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL) + if not downloadOnly: + for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version): + unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL) + unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL) + else: + print("\nLucene test done (--download-only specified)") print() print('Test Solr...') checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile) - for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version): - unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL) - solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version, - gitRevision, version, testArgs, baseURL) - - print() - print('Test Maven artifacts for Lucene and Solr...') - checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile) + if not downloadOnly: + for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version): + unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL) + solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version, + gitRevision, version, testArgs, baseURL) + print() + print('Test Maven artifacts for Lucene and Solr...') + checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile) + else: + print("Solr test done (--download-only specified)") print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))