From b013f9408691cceb7e8e7617bcaee0c0fc02c241 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 26 Jan 2022 20:56:35 +0100 Subject: [PATCH] Write XML test log files even for the flaky test re-runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the qt-testrunner script was avoiding to write XML log files for the flaky test re-runs, since parsing them always provides limited information about what went wrong (for example unparseable files) and might miss issues completely (for example a crashed test might not even write a log file), or even mislead if multiple different data points about a single testcase end up into the database.. But currently Coin is the system that uploads test information to the testresults database and expects to find such log files in an attempt to send as much information as possible regarding failures. Re-enabling these log files will allow Coin to deduce as much information as possible about the test runs. This is a temporary step at least, until the test-uploading logic is separated from Coin. Then it will be easier to find the full information about what happened with flaky or crashed tests by special logs from qt-testrunner.py, which is the test re-running orchestrator and therefore has more knowledge about what went wrong. Fixes: QTQAINFRA-4754 Change-Id: I69e4469cbe6c0a6ca5b2473eaf8d034632564342 Reviewed-by: Fabian Kosmale Reviewed-by: Toni Saario Reviewed-by: Simo Fält --- util/testrunner/qt-testrunner.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py index d0d54f054c..20ef2ccbb7 100755 --- a/util/testrunner/qt-testrunner.py +++ b/util/testrunner/qt-testrunner.py @@ -79,6 +79,7 @@ import argparse import subprocess import os import traceback +import time import timeit import xml.etree.ElementTree as ET import logging as L @@ -241,6 +242,10 @@ def run_test(arg_list: List[str], **kwargs): return proc +def unique_filename(test_basename: str) -> str: + timestamp = round(time.time() * 1000) + return f"{test_basename}-{timestamp}.xml" + # Returns tuple: (exit_code, xml_logfile) def run_full_test(test_basename, testargs: List[str], output_dir: str, no_extra_args=False, dryrun=False, @@ -253,7 +258,8 @@ def run_full_test(test_basename, testargs: List[str], output_dir: str, # Append arguments to write log to qtestlib XML file, # and text to stdout. if not no_extra_args: - results_files.append(os.path.join(output_dir, test_basename + ".xml")) + results_files.append( + os.path.join(output_dir, unique_filename(test_basename))) output_testargs.extend(["-o", results_files[0] + ",xml"]) output_testargs.extend(["-o", "-,txt"]) @@ -263,7 +269,8 @@ def run_full_test(test_basename, testargs: List[str], output_dir: str, return (proc.returncode, results_files[0] if results_files else None) -def rerun_failed_testcase(testargs: List[str], what_failed: WhatFailed, +def rerun_failed_testcase(test_basename, testargs: List[str], output_dir: str, + what_failed: WhatFailed, max_repeats, passes_needed, dryrun=False, timeout=None) -> bool: """Run a specific function:tag of a test, until it passes enough times, or @@ -276,13 +283,20 @@ def rerun_failed_testcase(testargs: List[str], what_failed: WhatFailed, if what_failed.tag: failed_arg += ":" + what_failed.tag + n_passes = 0 for i in range(max_repeats): + # For the individual testcase re-runs, we log to file since Coin needs + # to parse it. That is the reason we use unique filename every time. + output_args = [ + "-o", os.path.join(output_dir, unique_filename(test_basename)) + ",xml", + "-o", "-,txt"] L.info("Re-running testcase: %s", failed_arg) if i < max_repeats - 1: - proc = run_test(testargs + [failed_arg], timeout=timeout) + proc = run_test(testargs + output_args + [failed_arg], + timeout=timeout) else: # last re-run - proc = run_test(testargs + VERBOSE_ARGS + [failed_arg], + proc = run_test(testargs + output_args + VERBOSE_ARGS + [failed_arg], timeout=timeout, env={**os.environ, **VERBOSE_ENV}) if proc.returncode == 0: @@ -356,8 +370,8 @@ def main(): for what_failed in failed_functions: try: - ret = rerun_failed_testcase(args.testargs, what_failed, - args.max_repeats, args.passes_needed, + ret = rerun_failed_testcase(args.test_basename, args.testargs, args.log_dir, + what_failed, args.max_repeats, args.passes_needed, dryrun=args.dry_run, timeout=args.timeout) except Exception as e: L.exception("The test executable CRASHed uncontrollably!"