AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add QTest option for repeating the entire test execution

Browse Source 
Repeated test execution can be useful, under a debugger, to catch an
intermittent failure or, under memory instrumentation, to make memory
leaks easier to recognize.

The new -repeat flag allows running the entire test suite multiple times
within the same process. It works by executing all tests sequentially
before repeating the execution again.

This switch is a developer tool, and is not intended for CI. It can only
be used with the plain text logger.

Change-Id: I2439462c5c44d1c8aa3d3b5656de3eef44898c68
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
This commit is contained in:
Jøger Hansegård 2023-08-11 17:03:01 +02:00
parent f67499baab
commit 80a14c86b2
9 changed files with 84 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/testlib/doc/src/qttestlib-manual.qdoc
View File

@ -348,6 +348,10 @@
Disables the crash handler on Unix platforms.
On Windows, it re-enables the Windows Error Reporting dialog, which is
turned off by default. This is useful for debugging crashes.
\li \c -repeat \e n \br
Run the testsuite n times or until the test fails. Useful for finding
flaky tests. If negative, the tests are repeated forever. This is intended
as a developer tool, and is only supported with the plain text logger.
\li \c -platform \e name \br
This command line argument applies to all Qt applications, but might be

13
src/testlib/qabstracttestlogger.cpp
View File

@ -147,6 +147,19 @@ QAbstractTestLogger::~QAbstractTestLogger()
stream = nullptr;
}
/*!
Returns true if the logger supports repeated test runs.
Repetition of test runs is disabled by default, and can be enabled only for
test loggers that support it. Even if the logger may create syntactically
correct test reports, log-file analyzers may assume that test names are
unique within one report file.
*/
bool QAbstractTestLogger::isRepeatSupported() const
{
return false;
}
/*!
Returns true if the \c output stream is standard output.
*/

2
src/testlib/qabstracttestlogger_p.h
View File

@ -76,6 +76,8 @@ public:
virtual void addMessage(MessageTypes type, const QString &message,
const char *file = nullptr, int line = 0) = 0;
virtual bool isRepeatSupported() const;
bool isLoggingToStdout() const;
void outputString(const char *msg);

9
src/testlib/qplaintestlogger.cpp
View File

@ -492,4 +492,13 @@ void QPlainTestLogger::addMessage(MessageTypes type, const QString &message,
printMessage(MessageSource::Other, QTest::ptMessageType2String(type), qPrintable(message), file, line);
}
bool QPlainTestLogger::isRepeatSupported() const
{
// The plain text logger creates unstructured reports. Such reports are not
// parser friendly, and are unlikely to be parsed by any test reporting
// tools. We can therefore allow repeated test runs with minimum risk that
// any parsers fails to handle repeated test names.
return true;
}
QT_END_NAMESPACE

2
src/testlib/qplaintestlogger_p.h
View File

@ -43,6 +43,8 @@ public:
void addMessage(MessageTypes type, const QString &message,
const char *file = nullptr, int line = 0) override;
bool isRepeatSupported() const override;
private:
enum class MessageSource {
Incident,

35
src/testlib/qtestcase.cpp
View File

@ -539,6 +539,8 @@ static int eventDelay = -1;
static int timeout = -1;
#endif
static bool noCrashHandler = false;
static int repetitions = 1;
static bool repeatForever = false;
/*! \internal
Invoke a method of the object without generating warning if the method does not exist
@ -710,6 +712,9 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
int logFormat = -1; // Not set
const char *logFilename = nullptr;
repetitions = 1;
repeatForever = false;
QTest::testFunctions.clear();
QTest::testTags.clear();
@ -764,6 +769,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
" -maxwarnings n : Sets the maximum amount of messages to output.\n"
" 0 means unlimited, default: 2000\n"
" -nocrashhandler : Disables the crash handler. Useful for debugging crashes.\n"
" -repeat n : Run the testsuite n times or until the test fails.\n"
" Useful for finding flaky tests. If negative, the tests are\n"
" repeated forever. This is intended as a developer tool, and\n"
" is only supported with the plain text logger.\n"
"\n"
" Benchmarking options:\n"
#if QT_CONFIG(valgrind)
@ -913,6 +922,14 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
} else {
QTestLog::setMaxWarnings(qToInt(argv[++i]));
}
} else if (strcmp(argv[i], "-repeat") == 0) {
if (i + 1 >= argc) {
fprintf(stderr, "-repeat needs an extra parameter for the number of repetitions\n");
exit(1);
} else {
repetitions = qToInt(argv[++i]);
repeatForever = repetitions < 0;
}
} else if (strcmp(argv[i], "-nocrashhandler") == 0) {
QTest::noCrashHandler = true;
#if QT_CONFIG(valgrind)
@ -1066,6 +1083,11 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
if (addFallbackLogger)
QTestLog::addLogger(QTestLog::Plain, logFilename);
if (repetitions != 1 && !QTestLog::isRepeatSupported()) {
fprintf(stderr, "-repeat is only supported with plain text logger\n");
exit(1);
}
}
// Temporary, backwards compatibility, until qtdeclarative's use of it is converted
@ -2330,10 +2352,7 @@ void QTest::qInit(QObject *testObject, int argc, char **argv)
#if QT_CONFIG(valgrind)
if (QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindParentProcess)
#endif
{
QTestTable::globalTestTable();
QTestLog::startLogging();
}
}
/*! \internal
@ -2398,7 +2417,12 @@ int QTest::qRun()
return 1;
}
TestMethods test(currentTestObject, std::move(commandLineMethods));
test.invokeTests(currentTestObject);
while (QTestLog::failCount() == 0 && (repeatForever || repetitions-- > 0)) {
QTestTable::globalTestTable();
test.invokeTests(currentTestObject);
QTestTable::clearGlobalTestTable();
}
}
#ifndef QT_NO_EXCEPTIONS
@ -2435,10 +2459,7 @@ void QTest::qCleanup()
#if QT_CONFIG(valgrind)
if (QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindParentProcess)
#endif
{
QTestLog::stopLogging();
QTestTable::clearGlobalTestTable();
}
delete QBenchmarkGlobalData::current;
QBenchmarkGlobalData::current = nullptr;

15
src/testlib/qtestlog.cpp
View File

@ -559,6 +559,21 @@ bool QTestLog::hasLoggers()
return !QTest::loggers()->empty();
}
/*!
\internal
Returns true if all loggers support repeated test runs
*/
bool QTestLog::isRepeatSupported()
{
FOREACH_TEST_LOGGER {
if (!logger->isRepeatSupported())
return false;
}
return true;
}
bool QTestLog::loggerUsingStdout()
{
FOREACH_TEST_LOGGER {

1
src/testlib/qtestlog_p.h
View File

@ -91,6 +91,7 @@ public:
static void addLogger(QAbstractTestLogger *logger);
static bool hasLoggers();
static bool isRepeatSupported();
static bool loggerUsingStdout();
static void setVerboseLevel(int level);

10
tests/auto/testlib/selftests/tst_selftests.cpp
View File

@ -1233,6 +1233,7 @@ SCENARIO("Exit code is as expected")
{ 0, "globaldata testGlobal:global=true" },
{ 0, "globaldata testGlobal:local=true" },
{ 0, "globaldata testGlobal:global=true:local=true" },
{ 0, "globaldata testGlobal -repeat 2" },
{ 1, "globaldata testGlobal:local=true:global=true" },
{ 1, "globaldata testGlobal:global=true:blah" },
{ 1, "globaldata testGlobal:blah:local=true" },
@ -1244,6 +1245,15 @@ SCENARIO("Exit code is as expected")
{ 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" },
{ 1, "globaldata testGlobal:global=true skipSingle:blah" },
{ 2, "globaldata testGlobal:blah skipSingle:blue" },
// Passing -repeat argument
{ 1, "pass testNumber1 -repeat" },
{ 0, "pass testNumber1 -repeat 1" },
{ 0, "pass testNumber1 -repeat 1 -o out.xml,xml" },
{ 0, "pass testNumber1 -repeat 2" },
{ 0, "pass testNumber1 -repeat 2 -o -,txt" },
{ 0, "pass testNumber1 -repeat 2 -o -,txt -o log.txt,txt" },
{ 1, "pass testNumber1 -repeat 2 -o log.xml,xml" },
{ 1, "pass testNumber1 -repeat 2 -o -,txt -o -,xml" },
};
size_t n_testCases = sizeof(testCases) / sizeof(*testCases);