HBASE-16434
- Add date and count of flaky tests - Sort tests by decreasing order of flakyness - Internal links to each job's results - Correct calculation of total bad runs for a test - Fixes pylint errors Change-Id: I12ebc32ccec14c5ff389464b4de8ae93653c876c
This commit is contained in:
Apekshit Sharma
parent
741d0a4511
commit
0d6c4d92ed
|
|
@ -15,14 +15,18 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
##
|
||||
# script to find hanging test from Jenkins build output
|
||||
# usage: ./findHangingTests.py <url of Jenkins build console>
|
||||
#
|
||||
import re
|
||||
import requests
|
||||
import sys
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
# To disable 'invalid constant name' warnings.
|
||||
|
||||
"""
|
||||
# Script to find hanging test from Jenkins build output
|
||||
# usage: ./findHangingTests.py <url of Jenkins build console>
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import requests
|
||||
|
||||
# If any of these strings appear in the console output, it's a build one should probably ignore
|
||||
# for analyzing failed/hanging tests.
|
||||
|
|
@ -31,62 +35,64 @@ BAD_RUN_STRINGS = [
|
|||
"The forked VM terminated without properly saying goodbye", # JVM crashed.
|
||||
]
|
||||
|
||||
# Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully gets
|
||||
# the build information.
|
||||
# If there is error getting console text or if there are blacklisted strings in console text,
|
||||
# then returns None.
|
||||
# Definitions:
|
||||
# All tests: All testcases which were run.
|
||||
# Hanging test: A testcase which started but never finished.
|
||||
# Failed test: Testcase which encountered any kind of failure. It can be failing atomic tests,
|
||||
# timed out tests, etc
|
||||
# Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will be
|
||||
# included in failed tests.
|
||||
|
||||
def get_bad_tests(console_url):
|
||||
"""
|
||||
Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully gets
|
||||
the build information.
|
||||
If there is error getting console text or if there are blacklisted strings in console text,
|
||||
then returns None.
|
||||
"""
|
||||
response = requests.get(console_url)
|
||||
if response.status_code != 200:
|
||||
print "Error getting consoleText. Response = {} {}".format(
|
||||
response.status_code, response.reason)
|
||||
return
|
||||
|
||||
all_tests = set()
|
||||
hanging_tests = set()
|
||||
failed_tests = set()
|
||||
timeout_tests = set()
|
||||
# All tests: All testcases which were run.
|
||||
# Hanging test: A testcase which started but never finished.
|
||||
# Failed test: Testcase which encountered any kind of failure. It can be failing atomic tests,
|
||||
# timed out tests, etc
|
||||
# Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will be
|
||||
# included in failed tests.
|
||||
all_tests_set = set()
|
||||
hanging_tests_set = set()
|
||||
failed_tests_set = set()
|
||||
timeout_tests_set = set()
|
||||
for line in response.content.splitlines():
|
||||
result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", line)
|
||||
result1 = re.match("^Running org.apache.hadoop.hbase.(\\w*\\.)*(\\w*)", line)
|
||||
if result1:
|
||||
test_case = result1.group(2)
|
||||
if test_case in all_tests:
|
||||
print ("ERROR! Multiple tests with same name '{}'. Might get wrong results "
|
||||
if test_case in all_tests_set:
|
||||
print ("ERROR! Multiple tests with same name '{}'. Might get wrong results "
|
||||
"for this test.".format(test_case))
|
||||
else:
|
||||
hanging_tests.add(test_case)
|
||||
all_tests.add(test_case)
|
||||
result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\w*\.)*(\w*)", line)
|
||||
hanging_tests_set.add(test_case)
|
||||
all_tests_set.add(test_case)
|
||||
result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\\w*\\.)*(\\w*)", line)
|
||||
if result2:
|
||||
test_case = result2.group(2)
|
||||
if "FAILURE!" in line:
|
||||
failed_tests.add(test_case)
|
||||
if test_case not in hanging_tests:
|
||||
failed_tests_set.add(test_case)
|
||||
if test_case not in hanging_tests_set:
|
||||
print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results "
|
||||
"for this test.".format(test_case))
|
||||
else:
|
||||
hanging_tests.remove(test_case)
|
||||
result3 = re.match("^\s+(\w*).*\sTestTimedOut", line)
|
||||
hanging_tests_set.remove(test_case)
|
||||
result3 = re.match("^\\s+(\\w*).*\\sTestTimedOut", line)
|
||||
if result3:
|
||||
test_case = result3.group(1)
|
||||
timeout_tests.add(test_case)
|
||||
timeout_tests_set.add(test_case)
|
||||
for bad_string in BAD_RUN_STRINGS:
|
||||
if re.match(".*" + bad_string + ".*", line):
|
||||
print "Bad string found in build:\n > {}".format(line)
|
||||
return
|
||||
print "Result > total tests: {:4} failed : {:4} timedout : {:4} hanging : {:4}".format(
|
||||
len(all_tests), len(failed_tests), len(timeout_tests), len(hanging_tests))
|
||||
return [all_tests, failed_tests, timeout_tests, hanging_tests]
|
||||
len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set))
|
||||
return [all_tests_set, failed_tests_set, timeout_tests_set, hanging_tests_set]
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2 :
|
||||
if len(sys.argv) != 2:
|
||||
print "ERROR : Provide the jenkins job console URL as the only argument."
|
||||
sys.exit(1)
|
||||
|
||||
|
|
|
|||
|
|
@ -49,14 +49,35 @@
|
|||
Apache HBase Flaky Tests Dashboard
|
||||
</span>
|
||||
</p>
|
||||
<span>Last updated: <b>{{datetime}}</b></span><br>
|
||||
<span>Count of flaky tests (cumulated from all jobs):
|
||||
<b>{{bad_tests_count}}</b></span><br>
|
||||
<br><br>
|
||||
<span style="font-size:20px;"><b>List of Jobs</b></span><br>
|
||||
<br>
|
||||
{% set counter = 0 %}
|
||||
{% for url in results %}
|
||||
{% set counter = counter + 1 %}
|
||||
<a href="#job_{{ counter }}">{{ url |e }}</a>
|
||||
<br>
|
||||
{% endfor %}
|
||||
<br>
|
||||
<br>
|
||||
<span style="font-size:20px;"><b>Results</b></span><br>
|
||||
<br>
|
||||
{% set counter = 0 %}
|
||||
{% for url in results %}
|
||||
{% set result = results[url] %}
|
||||
{# Dedup ids since test names may duplicate across urls #}
|
||||
{% set counter = counter + 1 %}
|
||||
<span style="font-size:20px; font-weight:bold;">Job : {{ url |e }}
|
||||
<a href="{{ url |e }}" style="text-decoration:none;">🔗</a></span>
|
||||
<span id="job_{{ counter }}" style="font-weight:bold;">
|
||||
{{ url |e }}<br>
|
||||
<a href="{{ url |e }}">
|
||||
Go to <img height="16px" src="https://jenkins.io/sites/default/files/jenkins_favicon.ico">
|
||||
</a>
|
||||
|
||||
<a href="#">Go to top</a>
|
||||
</span>
|
||||
<br/><br/>
|
||||
<table>
|
||||
<tr>
|
||||
|
|
|
|||
|
|
@ -16,15 +16,24 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
# To disable 'invalid constant name' warnings.
|
||||
# pylint: disable=import-error
|
||||
# Testing environment may not have all dependencies.
|
||||
|
||||
"""
|
||||
This script uses Jenkins REST api to collect test result(s) of given build/builds and generates
|
||||
flakyness data about unittests.
|
||||
Print help: report-flakies.py -h
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from jinja2 import Template
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from jinja2 import Template
|
||||
|
||||
import requests
|
||||
|
||||
import findHangingTests
|
||||
|
|
@ -115,7 +124,7 @@ all_failed_tests = set()
|
|||
all_hanging_tests = set()
|
||||
# Contains { <url> : { <bad_test> : { 'all': [<build ids>], 'failed': [<build ids>],
|
||||
# 'timeout': [<build ids>], 'hanging': [<builds ids>] } } }
|
||||
url_to_bad_test_results = {}
|
||||
url_to_bad_test_results = OrderedDict()
|
||||
|
||||
# Iterates over each url, gets test results and prints flaky tests.
|
||||
expanded_urls = expand_multi_config_projects(args)
|
||||
|
|
@ -160,33 +169,48 @@ for url_max_build in expanded_urls:
|
|||
bad_tests.update(failed_tests.union(hanging_tests))
|
||||
|
||||
# For each bad test, get build ids where it ran, timed out, failed or hanged.
|
||||
test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(), 'hanging' : set()}
|
||||
test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(),
|
||||
'hanging' : set(), 'bad_count' : 0}
|
||||
for key in bad_tests}
|
||||
for build in build_id_to_results:
|
||||
[all_tests, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build]
|
||||
for bad_test in test_to_build_ids:
|
||||
is_bad = False
|
||||
if all_tests.issuperset([bad_test]):
|
||||
test_to_build_ids[bad_test]["all"].add(build)
|
||||
if timeout_tests.issuperset([bad_test]):
|
||||
test_to_build_ids[bad_test]['timeout'].add(build)
|
||||
is_bad = True
|
||||
if failed_tests.issuperset([bad_test]):
|
||||
test_to_build_ids[bad_test]['failed'].add(build)
|
||||
is_bad = True
|
||||
if hanging_tests.issuperset([bad_test]):
|
||||
test_to_build_ids[bad_test]['hanging'].add(build)
|
||||
url_to_bad_test_results[url] = test_to_build_ids
|
||||
is_bad = True
|
||||
if is_bad:
|
||||
test_to_build_ids[bad_test]['bad_count'] += 1
|
||||
|
||||
if len(test_to_build_ids) > 0:
|
||||
# Calculate flakyness % for each test.
|
||||
for bad_test in test_to_build_ids:
|
||||
test_to_build_ids[bad_test]['flakyness'] = (
|
||||
(test_to_build_ids[bad_test]['bad_count']) * 100.0 /
|
||||
len(test_to_build_ids[bad_test]['all']))
|
||||
|
||||
# Sort tests in descending order by flakyness.
|
||||
sorted_test_to_build_ids = OrderedDict(
|
||||
sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True))
|
||||
url_to_bad_test_results[url] = sorted_test_to_build_ids
|
||||
|
||||
if len(sorted_test_to_build_ids) > 0:
|
||||
print "URL: {}".format(url)
|
||||
print "{:>60} {:10} {:25} {}".format(
|
||||
"Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness")
|
||||
for bad_test in test_to_build_ids:
|
||||
failed = len(test_to_build_ids[bad_test]['failed'])
|
||||
timeout = len(test_to_build_ids[bad_test]['timeout'])
|
||||
hanging = len(test_to_build_ids[bad_test]['hanging'])
|
||||
total = len(test_to_build_ids[bad_test]['all'])
|
||||
for bad_test in sorted_test_to_build_ids:
|
||||
test_status = sorted_test_to_build_ids[bad_test]
|
||||
print "{:>60} {:10} {:7} ( {:4} / {:5} / {:5} ) {:2.0f}%".format(
|
||||
bad_test, total, failed + timeout, failed, timeout, hanging,
|
||||
(failed + timeout) * 100.0 / total)
|
||||
bad_test, len(test_status['all']), test_status['bad_count'],
|
||||
len(test_status['failed']), len(test_status['timeout']),
|
||||
len(test_status['hanging']), test_status['flakyness'])
|
||||
else:
|
||||
print "No flaky tests founds."
|
||||
if len(build_ids) == len(build_ids_without_tests_run):
|
||||
|
|
@ -218,4 +242,6 @@ with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"), "r") a
|
|||
template = Template(f.read())
|
||||
|
||||
with open("dashboard.html", "w") as f:
|
||||
f.write(template.render(results=url_to_bad_test_results))
|
||||
datetime = time.strftime("%m/%d/%Y %H:%M:%S")
|
||||
f.write(template.render(datetime=datetime, bad_tests_count=len(all_bad_tests),
|
||||
results=url_to_bad_test_results))
|
||||
|
|
|
|||
Loading…
Reference in New Issue