From 88e00d08894cd58f8cf1f51f9de6f36ce3183031 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Fri, 23 Feb 2018 19:20:14 -0500 Subject: [PATCH] LUCENE-8106: collect Ant test sysprops from Jenkins log and add them to test repro lines --- dev-tools/scripts/reproduceJenkinsFailures.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py index 775a22a9fc5..5e081cbd2d1 100644 --- a/dev-tools/scripts/reproduceJenkinsFailures.py +++ b/dev-tools/scripts/reproduceJenkinsFailures.py @@ -26,6 +26,17 @@ from textwrap import dedent # Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master) reGitRev = re.compile(r'Checking out Revision (\S+)\s+\(refs/remotes/origin/([^)]+)') +# Policeman Jenkins example: [Lucene-Solr-7.x-Linux] $ /var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/bin/ant "-Dargs=-XX:-UseCompressedOops -XX:+UseConcMarkSweepGC" jenkins-hourly +# Policeman Jenkins Windows example: [Lucene-Solr-master-Windows] $ cmd.exe /C "C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2\bin\ant.bat '"-Dargs=-client -XX:+UseConcMarkSweepGC"' jenkins-hourly && exit %%ERRORLEVEL%%" +# ASF Jenkins example: [Lucene-Solr-Tests-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant jenkins-hourly +# ASF Jenkins nightly example: [checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt jenkins-nightly +# ASF Jenkins smoker example: [Lucene-Solr-SmokeRelease-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant nightly-smoke +reAntInvocation = re.compile(r'\bant(?:\.bat)?\s+.*(?:jenkins-(?:hourly|nightly)|nightly-smoke)') +reAntSysprops = re.compile(r'"-D[^"]+"|-D[^=]+="[^"]*"|-D\S+') + +# sarowe Jenkins example: + export 'ANT_OPTS=-Xmx1150m -XX:+CMSClassUnloadingEnabled -Djava.awt.headless=true -Dargs="-Xmx1g"' +reAntOptions = re.compile(r"export\s+'?\s*ANT_OPTS=([^'\n\r]+)") + # Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8 # Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))') @@ -90,7 +101,9 @@ def run(cmd, rememberFailure=True): def fetchAndParseJenkinsLog(url): global revisionFromLog global branchFromLog + global antOptions revisionFromLog = None + antOptions = '' tests = {} print('[repro] Jenkins log URL: %s\n' % url) try: @@ -109,6 +122,16 @@ def fetchAndParseJenkinsLog(url): testcase = match.group(2) reproLineWithoutMethod = match.group(3).strip() tests[testcase] = reproLineWithoutMethod + else: + match = reAntInvocation.search(line) + if match is not None: + antOptions = ' '.join(reAntSysprops.findall(line)) + else: + match = reAntOptions.search(line) + if match is not None: + antOptions = ' '.join(reAntSysprops.findall(line)) + if len(antOptions) > 0: + print('[repro] Ant options: %s' % antOptions) except urllib.error.URLError as e: raise RuntimeError('ERROR: fetching %s : %s' % (url, e)) @@ -162,7 +185,7 @@ def groupTestsByModule(tests): def runTests(testIters, modules, tests): cwd = os.getcwd() - testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s' + testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s' for module in modules: moduleTests = list(modules[module]) testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests)) @@ -173,7 +196,7 @@ def runTests(testIters, modules, tests): try: if 0 != code: raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code)) - run(testCmdline % (testIters, testIters * numTests, testList, params)) + run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params)) finally: os.chdir(cwd)