qt5base-lts/util/testrunner/tests/qt_mock_test.py

#!/usr/bin/env python3
# Copyright (C) 2021 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0


# This is an artificial test, mimicking the Qt tests, for example tst_whatever.
# Its purpose is to assist in testing qt-testrunner.py.
#
# Mode A:
#
# If invoked with a test function argument, it runs that test function.
#
# Usage:
#
# $0 always_pass
# $0 always_fail
# $0 always_crash
# $0 fail_then_pass:N   # where N is the number of failing runs before passing
#
# Needs environment variable:
#   + QT_MOCK_TEST_STATE_FILE :: points to a unique filename, to be written
#     for keeping the state of the fail_then_pass:N tests.
#
# Mode B:
#
# If invoked without any argument, it runs the tests listed in the
# variable QT_MOCK_TEST_FAIL_LIST. If variable is empty it just runs
# the always_pass test. It also understands qtestlib's `-o outfile.xml,xml`
# option for writing a mock testlog in a file. Requires environment variables:
#   + QT_MOCK_TEST_STATE_FILE :: See above
#   + QT_MOCK_TEST_XML_TEMPLATE_FILE :: may point to the template XML file
#     located in the same source directory. Without this variable, the
#     option `-o outfile.xml,xml` will be ignored.
#   + QT_MOCK_TEST_FAIL_LIST :: may contain a comma-separated list of test
#     that should run.


import sys
import os
import traceback
from tst_testrunner import write_xml_log


MY_NAME         = os.path.basename(sys.argv[0])
STATE_FILE      = None
XML_TEMPLATE    = None
XML_OUTPUT_FILE = None


def put_failure(test_name):
    with open(STATE_FILE, "a") as f:
        f.write(test_name + "\n")
def get_failures(test_name):
    n = 0
    try:
        with open(STATE_FILE) as f:
            for line in f:
                if line.strip() == test_name:
                    n += 1
    except FileNotFoundError:
        return 0
    return n

# Only care about the XML log output file.
def parse_output_argument(a):
    global XML_OUTPUT_FILE
    if a.endswith(",xml"):
        XML_OUTPUT_FILE = a[:-4]

# Strip qtestlib specific arguments.
# Only care about the "-o ...,xml" argument.
def clean_cmdline():
    args = []
    prev_arg = None
    skip_next_arg = True    # Skip argv[0]
    for a in sys.argv:
        if skip_next_arg:
            if prev_arg == "-o":
                parse_output_argument(a)
            prev_arg = None
            skip_next_arg = False
            continue
        if a in ("-o", "-maxwarnings"):
            skip_next_arg = True
            prev_arg = a
            continue
        if a in ("-v1", "-v2", "-vs"):
            print("VERBOSE RUN")
            if "QT_LOGGING_RULES" in os.environ:
                print("Environment has QT_LOGGING_RULES:",
                      os.environ["QT_LOGGING_RULES"])
            continue
        args.append(a)
    return args


def log_test(testcase, result,
             testsuite=MY_NAME.rpartition(".")[0]):
    print("%-7s: %s::%s()" % (result, testsuite, testcase))

# Return the exit code
def run_test(testname):
    if   testname == "initTestCase":
        exit_code = 1              # specifically test that initTestCase fails
    elif testname == "always_pass":
        exit_code = 0
    elif testname == "always_fail":
        exit_code = 1
    elif testname == "always_crash":
        exit_code = 130
    elif testname.startswith("fail_then_pass"):
        wanted_fails   = int(testname.partition(":")[2])
        previous_fails = get_failures(testname)
        if previous_fails < wanted_fails:
            put_failure(testname)
            exit_code = 1
        else:
            exit_code = 0
    else:
        assert False, "Unknown argument: %s" % testname

    if exit_code == 0:
        log_test(testname, "PASS")
    elif exit_code == 1:
        log_test(testname, "FAIL!")
    else:
        log_test(testname, "CRASH!")

    return exit_code

def no_args_run():
    try:
        run_list = os.environ["QT_MOCK_TEST_RUN_LIST"].split(",")
    except KeyError:
        run_list = ["always_pass"]

    total_result = True
    fail_list = []
    for test in run_list:
        test_exit_code = run_test(test)
        if test_exit_code not in (0, 1):
            sys.exit(130)                                       # CRASH!
        if test_exit_code != 0:
            fail_list.append(test)
        total_result = total_result and (test_exit_code == 0)

    if XML_TEMPLATE and XML_OUTPUT_FILE:
        write_xml_log(XML_OUTPUT_FILE, failure=fail_list)

    if total_result:
        sys.exit(0)
    else:
        sys.exit(1)


def main():
    global STATE_FILE
    # Will fail if env var is not set.
    STATE_FILE = os.environ["QT_MOCK_TEST_STATE_FILE"]

    global XML_TEMPLATE
    if "QT_MOCK_TEST_XML_TEMPLATE_FILE" in os.environ:
        with open(os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]) as f:
            XML_TEMPLATE = f.read()

    args = clean_cmdline()

    if len(args) == 0:
        no_args_run()
        assert False, "Unreachable!"
    else:
        sys.exit(run_test(args[0]))


# TODO write XPASS test that does exit(1)

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        traceback.print_exc()
        exit(128)                      # Something went wrong with this script