mirror of https://github.com/apache/lucene.git
216 lines
8.4 KiB
Python
216 lines
8.4 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from textwrap import dedent
|
|
|
|
# Number of iterations per test suite
|
|
testIters = 5
|
|
|
|
usage = dedent('''\
|
|
Usage:\n
|
|
python3 -u %s URL\n
|
|
Must be run from a Lucene/Solr git workspace. Downloads the Jenkins
|
|
log pointed to by the given URL, parses it for Git revision and failed
|
|
Lucene/Solr tests, checks out the Git revision in the local workspace,
|
|
groups the failed tests by module, then runs
|
|
'ant test -Dtest.dups=%d -Dtests.class="*.test1[|*.test2[...]]" ...'
|
|
in each module of interest, failing at the end if any of the runs fails.
|
|
To control the maximum number of concurrent JVMs used for each module's
|
|
test run, set 'tests.jvms', e.g. in ~/lucene.build.properties
|
|
''' % (sys.argv[0], testIters))
|
|
|
|
reHelpArg = re.compile(r'-{1,2}(?:\?|h(?:elp)?)')
|
|
|
|
# Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
|
|
reGitRev = re.compile(r'Checking out Revision (\S+)')
|
|
|
|
# Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
|
|
# Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
|
|
reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))')
|
|
|
|
# Example: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21108/
|
|
reJenkinsURLWithoutConsoleText = re.compile(r'https?://.*/\d+/?\Z', re.IGNORECASE)
|
|
|
|
reJavaFile = re.compile(r'(.*)\.java\Z')
|
|
reModule = re.compile(r'\./(.*)/src/')
|
|
reTestOutputFile = re.compile(r'TEST-(.*\.([^-.]+))(?:-\d+)?\.xml\Z')
|
|
reErrorFailure = re.compile(r'(?:errors|failures)="[^0]')
|
|
|
|
# consoleText from Policeman Jenkins's Windows jobs fails to decode as UTF-8
|
|
encoding = 'iso-8859-1'
|
|
|
|
tests = {}
|
|
modules = {}
|
|
|
|
lastFailureCode = 0
|
|
gitCheckoutSucceeded = False
|
|
|
|
def runOutput(cmd):
|
|
print('[repro] %s' % cmd)
|
|
try:
|
|
return subprocess.check_output(cmd.split(' '), universal_newlines=True).strip()
|
|
except CalledProcessError as e:
|
|
raise RuntimeError("ERROR: Cmd '%s' failed with exit code %d and the following output:\n%s"
|
|
% (cmd, e.returncode, e.output))
|
|
|
|
# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False
|
|
def run(cmd, rememberFailure=True):
|
|
global lastFailureCode
|
|
print('[repro] %s' % cmd)
|
|
code = os.system(cmd)
|
|
if 0 != code and rememberFailure:
|
|
print('\n[repro] Setting last failure code to %d\n' % code)
|
|
lastFailureCode = code
|
|
return code
|
|
|
|
def fetchAndParseJenkinsLog(url):
|
|
global revision
|
|
revision = None
|
|
print('[repro] Jenkins log URL: %s\n' % url)
|
|
try:
|
|
with urllib.request.urlopen(url) as consoleText:
|
|
for rawLine in consoleText:
|
|
line = rawLine.decode(encoding)
|
|
match = reGitRev.match(line)
|
|
if match is not None:
|
|
revision = match.group(1)
|
|
print('[repro] Revision: %s\n' % revision)
|
|
else:
|
|
match = reReproLine.search(line)
|
|
if match is not None:
|
|
print('[repro] Repro line: %s\n' % match.group(1))
|
|
testcase = match.group(2)
|
|
reproLineWithoutMethod = match.group(3).strip()
|
|
tests[testcase] = reproLineWithoutMethod
|
|
except urllib.error.URLError as e:
|
|
raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
|
|
|
|
if revision == None:
|
|
if reJenkinsURLWithoutConsoleText.match(url):
|
|
print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n')
|
|
fetchAndParseJenkinsLog(url + '/consoleText')
|
|
else:
|
|
raise RuntimeError('ERROR: %s does not appear to be a Jenkins log.' % url)
|
|
if 0 == len(tests):
|
|
print('[repro] No "reproduce with" lines found; exiting.')
|
|
sys.exit(0)
|
|
|
|
def prepareWorkspace():
|
|
global gitCheckoutSucceeded
|
|
code = run('git checkout %s' % revision)
|
|
if 0 != code:
|
|
raise RuntimeError('ERROR: "git checkout %s" failed. See above. Maybe try "git pull"?' % revision)
|
|
gitCheckoutSucceeded = True
|
|
code = run('ant clean')
|
|
if 0 != code:
|
|
raise RuntimeError('ERROR: "ant clean" failed. See above.')
|
|
|
|
def groupTestsByModule():
|
|
for (dir, _, files) in os.walk('.'):
|
|
for file in files:
|
|
match = reJavaFile.search(file)
|
|
if match is not None:
|
|
test = match.group(1)
|
|
if test in tests:
|
|
match = reModule.match(dir)
|
|
module = match.group(1)
|
|
if module not in modules:
|
|
modules[module] = set()
|
|
modules[module].add(test)
|
|
print('[repro] Test suites by module:')
|
|
for module in modules:
|
|
print('[repro] %s' % module)
|
|
for test in modules[module]:
|
|
print('[repro] %s' % test)
|
|
|
|
def runTests():
|
|
global lastFailureCode
|
|
cwd = os.getcwd()
|
|
testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s'
|
|
for module in modules:
|
|
moduleTests = list(modules[module])
|
|
testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests))
|
|
numTests = len(moduleTests)
|
|
params = tests[moduleTests[0]] # Assumption: all tests in this module have the same cmdline params
|
|
os.chdir(module)
|
|
code = run('ant compile-test')
|
|
try:
|
|
if (0 != code):
|
|
raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
|
|
run(testCmdline % (testIters, testIters * numTests, testList, params))
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def printReport():
|
|
failures = {}
|
|
for start in ('lucene/build', 'solr/build'):
|
|
for (dir, _, files) in os.walk(start):
|
|
for file in files:
|
|
testOutputFileMatch = reTestOutputFile.search(file)
|
|
if testOutputFileMatch is not None:
|
|
testcase = testOutputFileMatch.group(1)
|
|
if testcase not in failures:
|
|
failures[testcase] = 0
|
|
with open(os.path.join(dir, file), encoding='UTF-8') as testOutputFile:
|
|
for line in testOutputFile:
|
|
errorFailureMatch = reErrorFailure.search(line)
|
|
if errorFailureMatch is not None:
|
|
failures[testcase] += 1
|
|
break
|
|
print("[repro] Failures:")
|
|
for testcase in sorted(failures):
|
|
print("[repro] %d/%d failed: %s" % (failures[testcase], testIters, testcase))
|
|
|
|
def rememberGitBranch():
|
|
global origGitBranch
|
|
origGitBranch = runOutput('git rev-parse --abbrev-ref HEAD')
|
|
if (origGitBranch == 'HEAD'): # In detached HEAD state
|
|
origGitBranch = runOutput('git rev-parse HEAD') # Use the SHA when not on a branch
|
|
print('[repro] Initial local git branch/revision: %s' % origGitBranch)
|
|
|
|
def main():
|
|
if 2 != len(sys.argv) or reHelpArg.match(sys.argv[1]):
|
|
print(usage)
|
|
sys.exit(0)
|
|
fetchAndParseJenkinsLog(sys.argv[1])
|
|
rememberGitBranch()
|
|
|
|
try:
|
|
prepareWorkspace()
|
|
groupTestsByModule()
|
|
runTests()
|
|
printReport()
|
|
except Exception as e:
|
|
print('[repro] %s' % e)
|
|
sys.exit(1)
|
|
finally:
|
|
if gitCheckoutSucceeded:
|
|
run('git checkout %s' % origGitBranch, rememberFailure=False) # Restore original git branch/sha
|
|
|
|
print('[repro] Exiting with code %d' % lastFailureCode)
|
|
sys.exit(lastFailureCode)
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
print('[repro] Keyboard interrupt...exiting')
|