diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py index 9ef87080494..28f48956dde 100755 --- a/dev-support/findHangingTests.py +++ b/dev-support/findHangingTests.py @@ -46,14 +46,22 @@ def get_bad_tests(console_url): result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) if result1: test_case = result1.group(2) - hanging_tests.add(test_case) - all_tests.add(test_case) + if test_case in all_tests: + print ("ERROR! Multiple tests with same name '{}'. Might get wrong results " + "for this test.".format(test_case)) + else: + hanging_tests.add(test_case) + all_tests.add(test_case) result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) if result2: test_case = result2.group(2) - hanging_tests.remove(test_case) if "FAILURE!" in line: failed_tests.add(test_case) + if test_case not in hanging_tests: + print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results " + "for this test.".format(test_case)) + else: + hanging_tests.remove(test_case) result3 = re.match("^\s+(\w*).*\sTestTimedOut", line) if result3: test_case = result3.group(1) diff --git a/dev-support/flaky-dashboard-template.html b/dev-support/flaky-dashboard-template.html new file mode 100644 index 00000000000..77dfc865c27 --- /dev/null +++ b/dev-support/flaky-dashboard-template.html @@ -0,0 +1,122 @@ + + + + + Apache HBase Flaky Dashboard + + + +

+ +      + + Apache HBase Flaky Tests Dashboard + +

+

+{% set counter = 0 %} +{% for url in results %} +{% set result = results[url] %} +{# Dedup ids since test names may duplicate across urls #} +{% set counter = counter + 1 %} + Job : {{ url |e }} + 🔗 +

+ + + + + + + + {% for test in result %} + {% set all = result[test]['all'] %} + {% set failed = result[test]['failed'] %} + {% set timeout = result[test]['timeout'] %} + {% set hanging = result[test]['hanging'] %} + {% set success = all.difference(failed).difference(hanging) %} + + + {% set flakyness = + (failed|length + hanging|length) * 100 / all|length %} + {% if flakyness == 100 %} + + + + + {% endfor %} +
Test NameFlakynessFailed/Timeout/HangingRun Ids
{{ test |e }} + {% else %} + + {% endif %} + {{ "{:.1f}% ({} / {})".format( + flakyness, failed|length + hanging|length, all|length) }} + + {{ failed|length }} / {{ timeout|length }} / {{ hanging|length }} + + {% set id = "details_" ~ test ~ "_" ~ counter %} + +
+ +
+


+{% endfor %} + + + diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py index c0d16c767d1..676eca32121 100755 --- a/dev-support/report-flakies.py +++ b/dev-support/report-flakies.py @@ -22,6 +22,7 @@ import argparse import findHangingTests from jinja2 import Template +import os import logging import requests @@ -177,114 +178,9 @@ if args.mvn: with open("./failed", "w") as file: file.write(",".join(all_failed_tests)) - -template = Template(""" - - - - Apache HBase Flaky Dashboard - - - -

- -      - - Apache HBase Flaky Tests Dashboard - -

-

- {% set counter = 0 %} - {% for url in results %} - {% set result = results[url] %} - {# Dedup ids since test names may duplicate across urls #} - {% set counter = counter + 1 %} - Job : {{ url |e }} - 🔗 -

- - - - - - - - {% for test in result %} - {% set all = result[test]['all'] %} - {% set failed = result[test]['failed'] %} - {% set timeout = result[test]['timeout'] %} - {% set hanging = result[test]['hanging'] %} - {% set success = all.difference(failed).difference(hanging) %} - - - {% set flakyness = - (failed|length + hanging|length) * 100 / all|length %} - {% if flakyness == 100 %} - - - - - {% endfor %} -
Test NameFlakynessFailed/Timeout/HangingRun Ids
{{ test |e }} - {% else %} - - {% endif %} - {{ "{:.1f}% ({} / {})".format( - flakyness, failed|length + hanging|length, all|length) }} - - {{ failed|length }} / {{ timeout|length }} / {{ hanging|length }} - - {% set id = "details_" ~ test ~ "_" ~ counter %} - -
- -
-


- {% endfor %} - - - - """) +dev_support_dir = os.path.dirname(os.path.abspath(__file__)) +with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"), "r") as f: + template = Template(f.read()) with open("dashboard.html", "w") as f: f.write(template.render(results=url_to_bad_test_results))