diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py index 2daf2e3770d..54275df849f 100755 --- a/dev-support/findHangingTests.py +++ b/dev-support/findHangingTests.py @@ -15,14 +15,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -## -# script to find hanging test from Jenkins build output -# usage: ./findHangingTests.py -# -import re -import requests -import sys +# pylint: disable=invalid-name +# To disable 'invalid constant name' warnings. + +""" +# Script to find hanging test from Jenkins build output +# usage: ./findHangingTests.py +""" + +import re +import sys +import requests # If any of these strings appear in the console output, it's a build one should probably ignore # for analyzing failed/hanging tests. @@ -31,62 +35,64 @@ BAD_RUN_STRINGS = [ "The forked VM terminated without properly saying goodbye", # JVM crashed. ] -# Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully gets -# the build information. -# If there is error getting console text or if there are blacklisted strings in console text, -# then returns None. -# Definitions: -# All tests: All testcases which were run. -# Hanging test: A testcase which started but never finished. -# Failed test: Testcase which encountered any kind of failure. It can be failing atomic tests, -# timed out tests, etc -# Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will be -# included in failed tests. + def get_bad_tests(console_url): + """ + Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully gets + the build information. + If there is error getting console text or if there are blacklisted strings in console text, + then returns None. + """ response = requests.get(console_url) if response.status_code != 200: print "Error getting consoleText. Response = {} {}".format( response.status_code, response.reason) return - all_tests = set() - hanging_tests = set() - failed_tests = set() - timeout_tests = set() + # All tests: All testcases which were run. + # Hanging test: A testcase which started but never finished. + # Failed test: Testcase which encountered any kind of failure. It can be failing atomic tests, + # timed out tests, etc + # Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will be + # included in failed tests. + all_tests_set = set() + hanging_tests_set = set() + failed_tests_set = set() + timeout_tests_set = set() for line in response.content.splitlines(): - result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) + result1 = re.match("^Running org.apache.hadoop.hbase.(\\w*\\.)*(\\w*)", line) if result1: test_case = result1.group(2) - if test_case in all_tests: - print ("ERROR! Multiple tests with same name '{}'. Might get wrong results " + if test_case in all_tests_set: + print ("ERROR! Multiple tests with same name '{}'. Might get wrong results " "for this test.".format(test_case)) else: - hanging_tests.add(test_case) - all_tests.add(test_case) - result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) + hanging_tests_set.add(test_case) + all_tests_set.add(test_case) + result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\\w*\\.)*(\\w*)", line) if result2: test_case = result2.group(2) if "FAILURE!" in line: - failed_tests.add(test_case) - if test_case not in hanging_tests: + failed_tests_set.add(test_case) + if test_case not in hanging_tests_set: print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results " "for this test.".format(test_case)) else: - hanging_tests.remove(test_case) - result3 = re.match("^\s+(\w*).*\sTestTimedOut", line) + hanging_tests_set.remove(test_case) + result3 = re.match("^\\s+(\\w*).*\\sTestTimedOut", line) if result3: test_case = result3.group(1) - timeout_tests.add(test_case) + timeout_tests_set.add(test_case) for bad_string in BAD_RUN_STRINGS: if re.match(".*" + bad_string + ".*", line): print "Bad string found in build:\n > {}".format(line) return print "Result > total tests: {:4} failed : {:4} timedout : {:4} hanging : {:4}".format( - len(all_tests), len(failed_tests), len(timeout_tests), len(hanging_tests)) - return [all_tests, failed_tests, timeout_tests, hanging_tests] + len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set)) + return [all_tests_set, failed_tests_set, timeout_tests_set, hanging_tests_set] if __name__ == "__main__": - if len(sys.argv) != 2 : + if len(sys.argv) != 2: print "ERROR : Provide the jenkins job console URL as the only argument." sys.exit(1) diff --git a/dev-support/flaky-dashboard-template.html b/dev-support/flaky-dashboard-template.html index 77dfc865c27..70febb48a5e 100644 --- a/dev-support/flaky-dashboard-template.html +++ b/dev-support/flaky-dashboard-template.html @@ -49,14 +49,35 @@ Apache HBase Flaky Tests Dashboard

+Last updated: {{datetime}}
+Count of flaky tests (cumulated from all jobs): + {{bad_tests_count}}


+List of Jobs
+
+{% set counter = 0 %} +{% for url in results %} +{% set counter = counter + 1 %} +{{ url |e }} +
+{% endfor %} +
+
+Results
+
{% set counter = 0 %} {% for url in results %} {% set result = results[url] %} {# Dedup ids since test names may duplicate across urls #} {% set counter = counter + 1 %} - Job : {{ url |e }} - 🔗 + + {{ url |e }}
+ + Go to + +      + Go to top +

diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py index 8c93b0b84cf..92b78cc185d 100755 --- a/dev-support/report-flakies.py +++ b/dev-support/report-flakies.py @@ -16,15 +16,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=invalid-name +# To disable 'invalid constant name' warnings. +# pylint: disable=import-error +# Testing environment may not have all dependencies. + """ This script uses Jenkins REST api to collect test result(s) of given build/builds and generates flakyness data about unittests. Print help: report-flakies.py -h """ + import argparse -from jinja2 import Template import logging import os +import time +from collections import OrderedDict +from jinja2 import Template + import requests import findHangingTests @@ -115,7 +124,7 @@ all_failed_tests = set() all_hanging_tests = set() # Contains { : { : { 'all': [], 'failed': [], # 'timeout': [], 'hanging': [] } } } -url_to_bad_test_results = {} +url_to_bad_test_results = OrderedDict() # Iterates over each url, gets test results and prints flaky tests. expanded_urls = expand_multi_config_projects(args) @@ -160,33 +169,48 @@ for url_max_build in expanded_urls: bad_tests.update(failed_tests.union(hanging_tests)) # For each bad test, get build ids where it ran, timed out, failed or hanged. - test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(), 'hanging' : set()} + test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(), + 'hanging' : set(), 'bad_count' : 0} for key in bad_tests} for build in build_id_to_results: [all_tests, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build] for bad_test in test_to_build_ids: + is_bad = False if all_tests.issuperset([bad_test]): test_to_build_ids[bad_test]["all"].add(build) if timeout_tests.issuperset([bad_test]): test_to_build_ids[bad_test]['timeout'].add(build) + is_bad = True if failed_tests.issuperset([bad_test]): test_to_build_ids[bad_test]['failed'].add(build) + is_bad = True if hanging_tests.issuperset([bad_test]): test_to_build_ids[bad_test]['hanging'].add(build) - url_to_bad_test_results[url] = test_to_build_ids + is_bad = True + if is_bad: + test_to_build_ids[bad_test]['bad_count'] += 1 - if len(test_to_build_ids) > 0: + # Calculate flakyness % for each test. + for bad_test in test_to_build_ids: + test_to_build_ids[bad_test]['flakyness'] = ( + (test_to_build_ids[bad_test]['bad_count']) * 100.0 / + len(test_to_build_ids[bad_test]['all'])) + + # Sort tests in descending order by flakyness. + sorted_test_to_build_ids = OrderedDict( + sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True)) + url_to_bad_test_results[url] = sorted_test_to_build_ids + + if len(sorted_test_to_build_ids) > 0: print "URL: {}".format(url) print "{:>60} {:10} {:25} {}".format( "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness") - for bad_test in test_to_build_ids: - failed = len(test_to_build_ids[bad_test]['failed']) - timeout = len(test_to_build_ids[bad_test]['timeout']) - hanging = len(test_to_build_ids[bad_test]['hanging']) - total = len(test_to_build_ids[bad_test]['all']) + for bad_test in sorted_test_to_build_ids: + test_status = sorted_test_to_build_ids[bad_test] print "{:>60} {:10} {:7} ( {:4} / {:5} / {:5} ) {:2.0f}%".format( - bad_test, total, failed + timeout, failed, timeout, hanging, - (failed + timeout) * 100.0 / total) + bad_test, len(test_status['all']), test_status['bad_count'], + len(test_status['failed']), len(test_status['timeout']), + len(test_status['hanging']), test_status['flakyness']) else: print "No flaky tests founds." if len(build_ids) == len(build_ids_without_tests_run): @@ -218,4 +242,6 @@ with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"), "r") a template = Template(f.read()) with open("dashboard.html", "w") as f: - f.write(template.render(results=url_to_bad_test_results)) + datetime = time.strftime("%m/%d/%Y %H:%M:%S") + f.write(template.render(datetime=datetime, bad_tests_count=len(all_bad_tests), + results=url_to_bad_test_results))