#!/usr/bin/env python ## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script uses Jenkins REST api to collect test result(s) of given build/builds and generates # flakyness data about unittests. # Print help: report-flakies.py -h import argparse import findHangingTests import logging import requests import sys parser = argparse.ArgumentParser() parser.add_argument("--urls", metavar="url[ max-builds]", action="append", required=True, help="Urls to analyze, which can refer to simple projects, multi-configuration projects or " "individual build run. Optionally, specify maximum builds to analyze for this url " "(if available on jenkins) using space as separator. By default, all available " "builds are analyzed.") parser.add_argument("--mvn", action="store_true", help="Writes two strings for including/excluding these flaky tests using maven flags. These " "strings are written to files so they can be saved as artifacts and easily imported in " "other projects. Also writes timeout and failing tests in separate files for " "reference.") parser.add_argument("-v", "--verbose", help="Prints more logs.", action="store_true") args = parser.parse_args() logging.basicConfig() logger = logging.getLogger(__name__) if args.verbose: logger.setLevel(logging.INFO) # Given url of an executed build, analyzes its console text, and returns # [list of all tests, list of timeout tests, list of failed tests]. def get_bad_tests(build_url): logger.info("Getting test results for %s", build_url) console_url = build_url + "/consoleText" return findHangingTests.get_hanging_tests(console_url) # If any url is of type multi-configuration project (i.e. has key 'activeConfigurations'), # get urls for individual jobs. def expand_multi_configuration_projects(urls_list): expanded_urls = [] for url_max_build in urls_list: splits = url_max_build.split() url = splits[0] max_builds = 10000 # Some high value if len(splits) == 2: max_builds = int(splits[1]) json_response = requests.get(url + "/api/json").json() if json_response.has_key("activeConfigurations"): for config in json_response["activeConfigurations"]: expanded_urls.append({'url':config["url"], 'max_builds': max_builds}) else: expanded_urls.append({'url':url, 'max_builds': max_builds}) return expanded_urls # Set of timeout/failed tests across all given urls. all_timeout_tests = set() all_failed_tests = set() # Iterates over each url, gets test results and prints flaky tests. expanded_urls = expand_multi_configuration_projects(args.urls) for url_max_build in expanded_urls: url = url_max_build["url"] json_response = requests.get(url + "/api/json").json() if json_response.has_key("builds"): builds = json_response["builds"] logger.info("Analyzing job: %s", url) else: builds = [{'number' : json_response["id"], 'url': url}] logger.info("Analyzing build : %s", url) build_id_to_results = {} num_builds = 0 build_ids = [] build_ids_without_tests_run = [] for build in builds: build_id = build["number"] build_ids.append(build_id) result = get_bad_tests(build["url"]) if len(result[0]) > 0: build_id_to_results[build_id] = result else: build_ids_without_tests_run.append(build_id) num_builds += 1 if num_builds == url_max_build["max_builds"]: break # Collect list of bad tests. bad_tests = set() for build in build_id_to_results: [_, timeout_tests, failed_tests] = build_id_to_results[build] all_timeout_tests.update(timeout_tests) all_failed_tests.update(failed_tests) bad_tests.update(timeout_tests.union(failed_tests)) # Get total and failed/timeout times for each bad test. build_counts = {key : {'total': 0, 'timeout': 0, 'fail': 0 } for key in bad_tests} for build in build_id_to_results: [all_tests, timeout_tests, failed_tests] = build_id_to_results[build] for bad_test in bad_tests: if all_tests.issuperset([bad_test]): build_counts[bad_test]["total"] += 1 if timeout_tests.issuperset([bad_test]): build_counts[bad_test]['timeout'] += 1 if failed_tests.issuperset([bad_test]): build_counts[bad_test]['fail'] += 1 if len(bad_tests) > 0: print "URL: {}".format(url) print "{:>60} {:25} {:10} {}".format( "Test Name", "Bad Runs(failed/timeout)", "Total Runs", "Flakyness") for bad_test in bad_tests: fail = build_counts[bad_test]['fail'] timeout = build_counts[bad_test]['timeout'] total = build_counts[bad_test]['total'] print "{:>60} {:10} ({:4} / {:4}) {:10} {:2.0f}%".format( bad_test, fail + timeout, fail, timeout, total, (fail + timeout) * 100.0 / total) else: print "No flaky tests founds." if len(build_ids) == len(build_ids_without_tests_run): print "None of the analyzed builds have test result." print "Builds analyzed: {}".format(build_ids) print "Builds without any test runs: {}".format(build_ids_without_tests_run) print "" all_bad_tests = all_timeout_tests.union(all_failed_tests) if args.mvn: includes = ",".join(all_bad_tests) with open("./includes", "w") as inc_file: inc_file.write(includes) excludes = ["**/{0}.java".format(bad_test) for bad_test in all_bad_tests] with open("./excludes", "w") as exc_file: exc_file.write(",".join(excludes)) with open("./timeout", "w") as file: file.write(",".join(all_timeout_tests)) with open("./failed", "w") as file: file.write(",".join(all_failed_tests))