mirror of https://github.com/apache/lucene.git
315 lines
14 KiB
Python
315 lines
14 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import http.client
|
|
import os
|
|
import re
|
|
import shutil
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import urllib.error
|
|
import urllib.request
|
|
from textwrap import dedent
|
|
|
|
# Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
|
|
reGitRev = re.compile(r'Checking out Revision (\S+)\s+\(refs/remotes/origin/([^)]+)')
|
|
|
|
# Policeman Jenkins example: [Lucene-Solr-7.x-Linux] $ /var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/bin/ant "-Dargs=-XX:-UseCompressedOops -XX:+UseConcMarkSweepGC" jenkins-hourly
|
|
# Policeman Jenkins Windows example: [Lucene-Solr-master-Windows] $ cmd.exe /C "C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2\bin\ant.bat '"-Dargs=-client -XX:+UseConcMarkSweepGC"' jenkins-hourly && exit %%ERRORLEVEL%%"
|
|
# ASF Jenkins example: [Lucene-Solr-Tests-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant jenkins-hourly
|
|
# ASF Jenkins nightly example: [checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt jenkins-nightly
|
|
# ASF Jenkins smoker example: [Lucene-Solr-SmokeRelease-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant nightly-smoke
|
|
reAntInvocation = re.compile(r'\bant(?:\.bat)?\s+.*(?:jenkins-(?:hourly|nightly)|nightly-smoke)')
|
|
reAntSysprops = re.compile(r'"-D[^"]+"|-D[^=]+="[^"]*"|-D\S+')
|
|
|
|
# Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
|
|
# Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
|
|
reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))')
|
|
reTestsSeed = re.compile(r'-Dtests.seed=\S+\s*')
|
|
|
|
# Example: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21108/
|
|
reJenkinsURLWithoutConsoleText = re.compile(r'https?://.*/\d+/?\Z', re.IGNORECASE)
|
|
|
|
reJavaFile = re.compile(r'(.*)\.java\Z')
|
|
reModule = re.compile(r'\.[\\/](.*)[\\/]src[\\/]')
|
|
reTestOutputFile = re.compile(r'TEST-(.*\.([^-.]+))(?:-\d+)?\.xml\Z')
|
|
reErrorFailure = re.compile(r'(?:errors|failures)="[^0]')
|
|
reGitMainBranch = re.compile(r'^(?:master|branch_[x_\d]+)$')
|
|
|
|
# consoleText from Policeman Jenkins's Windows jobs fails to decode as UTF-8
|
|
encoding = 'iso-8859-1'
|
|
|
|
lastFailureCode = 0
|
|
gitCheckoutSucceeded = False
|
|
|
|
description = dedent('''\
|
|
Must be run from a Lucene/Solr git workspace. Downloads the Jenkins
|
|
log pointed to by the given URL, parses it for Git revision and failed
|
|
Lucene/Solr tests, checks out the Git revision in the local workspace,
|
|
groups the failed tests by module, then runs
|
|
'ant test -Dtest.dups=%d -Dtests.class="*.test1[|*.test2[...]]" ...'
|
|
in each module of interest, failing at the end if any of the runs fails.
|
|
To control the maximum number of concurrent JVMs used for each module's
|
|
test run, set 'tests.jvms', e.g. in ~/lucene.build.properties
|
|
''')
|
|
defaultIters = 5
|
|
|
|
def readConfig():
|
|
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
description=description)
|
|
parser.add_argument('url', metavar='URL',
|
|
help='Points to the Jenkins log to parse')
|
|
parser.add_argument('--no-git', dest='useGit', action='store_false', default=True,
|
|
help='Do not run "git" at all')
|
|
parser.add_argument('--iters', dest='testIters', type=int, default=defaultIters, metavar='N',
|
|
help='Number of iterations per test suite (default: %d)' % defaultIters)
|
|
return parser.parse_args()
|
|
|
|
def runOutput(cmd):
|
|
print('[repro] %s' % cmd)
|
|
try:
|
|
return subprocess.check_output(cmd.split(' '), universal_newlines=True).strip()
|
|
except CalledProcessError as e:
|
|
raise RuntimeError("ERROR: Cmd '%s' failed with exit code %d and the following output:\n%s"
|
|
% (cmd, e.returncode, e.output))
|
|
|
|
# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False
|
|
def run(cmd, rememberFailure=True):
|
|
global lastFailureCode
|
|
print('[repro] %s' % cmd)
|
|
code = os.system(cmd)
|
|
if 0 != code and rememberFailure:
|
|
print('\n[repro] Setting last failure code to %d\n' % code)
|
|
lastFailureCode = code
|
|
return code
|
|
|
|
def fetchAndParseJenkinsLog(url, numRetries):
|
|
global revisionFromLog
|
|
global branchFromLog
|
|
global antOptions
|
|
revisionFromLog = None
|
|
antOptions = ''
|
|
tests = {}
|
|
print('[repro] Jenkins log URL: %s\n' % url)
|
|
try:
|
|
# HTTPS fails at certificate validation, see LUCENE-9412, PEP-476
|
|
context = ssl._create_unverified_context()
|
|
with urllib.request.urlopen(url, context=context) as consoleText:
|
|
for rawLine in consoleText:
|
|
line = rawLine.decode(encoding)
|
|
match = reGitRev.match(line)
|
|
if match is not None:
|
|
revisionFromLog = match.group(1)
|
|
branchFromLog = match.group(2)
|
|
print('[repro] Revision: %s\n' % revisionFromLog)
|
|
else:
|
|
match = reReproLine.search(line)
|
|
if match is not None:
|
|
print('[repro] Repro line: %s\n' % match.group(1))
|
|
testcase = match.group(2)
|
|
reproLineWithoutMethod = match.group(3).strip()
|
|
tests[testcase] = reproLineWithoutMethod
|
|
else:
|
|
match = reAntInvocation.search(line)
|
|
if match is not None:
|
|
antOptions = ' '.join(reAntSysprops.findall(line))
|
|
if len(antOptions) > 0:
|
|
print('[repro] Ant options: %s' % antOptions)
|
|
except urllib.error.URLError as e:
|
|
raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
|
|
except http.client.IncompleteRead as e:
|
|
if numRetries > 0:
|
|
print('[repro] Encountered IncompleteRead exception, pausing and then retrying...')
|
|
time.sleep(2) # pause for 2 seconds
|
|
return fetchAndParseJenkinsLog(url, numRetries - 1)
|
|
else:
|
|
print('[repro] Encountered IncompleteRead exception, aborting after too many retries.')
|
|
raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
|
|
|
|
if revisionFromLog == None:
|
|
if reJenkinsURLWithoutConsoleText.match(url):
|
|
print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n')
|
|
return fetchAndParseJenkinsLog(url + '/consoleText', numRetries)
|
|
else:
|
|
raise RuntimeError('ERROR: %s does not appear to be a Jenkins log.' % url)
|
|
if 0 == len(tests):
|
|
print('[repro] No "reproduce with" lines found; exiting.')
|
|
sys.exit(0)
|
|
return tests
|
|
|
|
def prepareWorkspace(useGit, gitRef):
|
|
global gitCheckoutSucceeded
|
|
if useGit:
|
|
code = run('git fetch')
|
|
if 0 != code:
|
|
raise RuntimeError('ERROR: "git fetch" failed. See above.')
|
|
checkoutCmd = 'git checkout %s' % gitRef
|
|
code = run(checkoutCmd)
|
|
if 0 != code:
|
|
addWantedBranchCmd = "git remote set-branches --add origin %s" % gitRef
|
|
checkoutBranchCmd = 'git checkout -t -b %s origin/%s' % (gitRef, gitRef) # Checkout remote branch as new local branch
|
|
print('"%s" failed. Trying "%s" and "%s".' % (checkoutCmd, addWantedBranchCmd, checkoutBranchCmd))
|
|
code = run(addWantedBranchCmd)
|
|
if 0 != code:
|
|
raise RuntimeError('ERROR: "%s" failed. See above.' % addWantedBranchCmd)
|
|
code = run(checkoutBranchCmd)
|
|
if 0 != code:
|
|
raise RuntimeError('ERROR: "%s" failed. See above.' % checkoutBranchCmd)
|
|
gitCheckoutSucceeded = True
|
|
run('git merge --ff-only', rememberFailure=False) # Ignore failure on non-branch ref
|
|
|
|
code = run('ant clean')
|
|
if 0 != code:
|
|
raise RuntimeError('ERROR: "ant clean" failed. See above.')
|
|
|
|
def groupTestsByModule(tests):
|
|
modules = {}
|
|
for (dir, _, files) in os.walk('.'):
|
|
for file in files:
|
|
match = reJavaFile.search(file)
|
|
if match is not None:
|
|
test = match.group(1)
|
|
if test in tests:
|
|
match = reModule.match(dir)
|
|
module = match.group(1)
|
|
if module not in modules:
|
|
modules[module] = set()
|
|
modules[module].add(test)
|
|
print('[repro] Test suites by module:')
|
|
for module in modules:
|
|
print('[repro] %s' % module)
|
|
for test in modules[module]:
|
|
print('[repro] %s' % test)
|
|
return modules
|
|
|
|
def runTests(testIters, modules, tests):
|
|
cwd = os.getcwd()
|
|
testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'
|
|
for module in modules:
|
|
moduleTests = list(modules[module])
|
|
testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests))
|
|
numTests = len(moduleTests)
|
|
params = tests[moduleTests[0]] # Assumption: all tests in this module have the same cmdline params
|
|
os.chdir(module)
|
|
code = run('ant compile-test')
|
|
try:
|
|
if 0 != code:
|
|
raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
|
|
run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params))
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def printAndMoveReports(testIters, newSubDir, location):
|
|
failures = {}
|
|
for start in ('lucene/build', 'solr/build'):
|
|
for (dir, _, files) in os.walk(start):
|
|
for file in files:
|
|
testOutputFileMatch = reTestOutputFile.search(file)
|
|
if testOutputFileMatch is not None:
|
|
testcase = testOutputFileMatch.group(1)
|
|
if testcase not in failures:
|
|
failures[testcase] = 0
|
|
filePath = os.path.join(dir, file)
|
|
with open(filePath, encoding='UTF-8') as testOutputFile:
|
|
for line in testOutputFile:
|
|
errorFailureMatch = reErrorFailure.search(line)
|
|
if errorFailureMatch is not None:
|
|
failures[testcase] += 1
|
|
break
|
|
# have to play nice with 'ant clean'...
|
|
newDirPath = os.path.join('repro-reports', newSubDir, dir)
|
|
os.makedirs(newDirPath, exist_ok=True)
|
|
os.rename(filePath, os.path.join(newDirPath, file))
|
|
print("[repro] Failures%s:" % location)
|
|
for testcase in sorted(failures, key=lambda t: (failures[t],t)): # sort by failure count, then by testcase
|
|
print("[repro] %d/%d failed: %s" % (failures[testcase], testIters, testcase))
|
|
return failures
|
|
|
|
def getLocalGitBranch():
|
|
origGitBranch = runOutput('git rev-parse --abbrev-ref HEAD')
|
|
if origGitBranch == 'HEAD': # In detached HEAD state
|
|
origGitBranch = runOutput('git rev-parse HEAD') # Use the SHA when not on a branch
|
|
print('[repro] Initial local git branch/revision: %s' % origGitBranch)
|
|
return origGitBranch
|
|
|
|
def main():
|
|
config = readConfig()
|
|
tests = fetchAndParseJenkinsLog(config.url, numRetries = 2)
|
|
if config.useGit:
|
|
localGitBranch = getLocalGitBranch()
|
|
|
|
try:
|
|
# have to play nice with ant clean, so printAndMoveReports will move all the junit XML files here...
|
|
print('[repro] JUnit rest result XML files will be moved to: ./repro-reports')
|
|
if os.path.isdir('repro-reports'):
|
|
print('[repro] Deleting old ./repro-reports');
|
|
shutil.rmtree('repro-reports')
|
|
prepareWorkspace(config.useGit, revisionFromLog)
|
|
modules = groupTestsByModule(tests)
|
|
runTests(config.testIters, modules, tests)
|
|
failures = printAndMoveReports(config.testIters, 'orig',
|
|
' w/original seeds' + (' at %s' % revisionFromLog if config.useGit else ''))
|
|
|
|
|
|
if config.useGit:
|
|
# Retest 100% failures at the tip of the branch
|
|
oldTests = tests
|
|
tests = {}
|
|
for fullClass in failures:
|
|
testcase = fullClass[(fullClass.rindex('.') + 1):]
|
|
if failures[fullClass] == config.testIters:
|
|
tests[testcase] = oldTests[testcase]
|
|
if len(tests) > 0:
|
|
print('\n[repro] Re-testing 100%% failures at the tip of %s' % branchFromLog)
|
|
prepareWorkspace(True, branchFromLog)
|
|
modules = groupTestsByModule(tests)
|
|
runTests(config.testIters, modules, tests)
|
|
failures = printAndMoveReports(config.testIters, 'branch-tip',
|
|
' original seeds at the tip of %s' % branchFromLog)
|
|
|
|
# Retest 100% tip-of-branch failures without a seed
|
|
oldTests = tests
|
|
tests = {}
|
|
for fullClass in failures:
|
|
testcase = fullClass[(fullClass.rindex('.') + 1):]
|
|
if failures[fullClass] == config.testIters:
|
|
tests[testcase] = re.sub(reTestsSeed, '', oldTests[testcase])
|
|
if len(tests) > 0:
|
|
print('\n[repro] Re-testing 100%% failures at the tip of %s without a seed' % branchFromLog)
|
|
prepareWorkspace(False, branchFromLog)
|
|
modules = groupTestsByModule(tests)
|
|
runTests(config.testIters, modules, tests)
|
|
printAndMoveReports(config.testIters, 'branch-tip-no-seed',
|
|
' at the tip of %s without a seed' % branchFromLog)
|
|
except Exception as e:
|
|
print('[repro] %s' % traceback.format_exc())
|
|
sys.exit(1)
|
|
finally:
|
|
if config.useGit and gitCheckoutSucceeded:
|
|
run('git checkout %s' % localGitBranch, rememberFailure=False) # Restore original git branch/sha
|
|
|
|
print('[repro] Exiting with code %d' % lastFailureCode)
|
|
sys.exit(lastFailureCode)
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
print('[repro] Keyboard interrupt...exiting')
|