AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

qtestlib: Add option to skip blacklisted tests

Browse Source 
Add a command-line option "-skipblacklisted" to testlib to skip
blacklisted test cases. Currently, blacklisted test cases are run,
but results are ignored. For test code coverage measurements, it's
important to see the code coverage of the actually tested code in
comparison to the code that was run but not actually tested.

The default approach remains unchanged, meaning that blacklisted
tests are run with the results ignored.

Fixes: QTBUG-112793
Change-Id: I6fe0a6353cb1c021e0232c79bb4f404632fb0bce
Reviewed-by: Jason McDonald <macadder1@gmail.com>
This commit is contained in:
Kalle Viironen 2023-11-21 09:45:46 +02:00
parent 0b60450eee
commit a397247e2b
13 changed files with 246 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/testlib/doc/src/qttestlib-manual.qdoc
View File

@ -352,6 +352,12 @@
Run the testsuite n times or until the test fails. Useful for finding
flaky tests. If negative, the tests are repeated forever. This is intended
as a developer tool, and is only supported with the plain text logger.
\li \c -skipblacklisted \br
Skip the blacklisted tests. This option is intended to allow more accurate
measurement of test coverage by preventing blacklisted tests from inflating
coverage statistics. When not measuring test coverage, it is recommended to
execute blacklisted tests to reveal any changes in their results, such as
a new crash or the issue that caused blacklisting being resolved.
\li \c -platform \e name \br
This command line argument applies to all Qt applications, but might be

35
src/testlib/qtestcase.cpp
View File

@ -541,6 +541,7 @@ static int timeout = -1;
static bool noCrashHandler = false;
static int repetitions = 1;
static bool repeatForever = false;
static bool skipBlacklisted = false;
/*! \internal
Invoke a method of the object without generating warning if the method does not exist
@ -773,6 +774,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
" Useful for finding flaky tests. If negative, the tests are\n"
" repeated forever. This is intended as a developer tool, and\n"
" is only supported with the plain text logger.\n"
" -skipblacklisted : Skip blacklisted tests. Useful for measuring test coverage.\n"
"\n"
" Benchmarking options:\n"
#if QT_CONFIG(valgrind)
@ -932,6 +934,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
}
} else if (strcmp(argv[i], "-nocrashhandler") == 0) {
QTest::noCrashHandler = true;
} else if (strcmp(argv[i], "-skipblacklisted") == 0) {
QTest::skipBlacklisted = true;
#if QT_CONFIG(valgrind)
} else if (strcmp(argv[i], "-callgrind") == 0) {
if (!QBenchmarkValgrindUtils::haveValgrind()) {
@ -1417,6 +1421,7 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
tag[global.size()] == ':';
};
bool foundFunction = false;
bool blacklisted = false;
/* For each entry in the global data table, do: */
do {
@ -1443,20 +1448,28 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)),
QLatin1StringView(globalDataTag(curGlobalDataIndex)))) {
foundFunction = true;
if (QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex),
globalDataTag(curGlobalDataIndex))) {
blacklisted = QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex),
globalDataTag(curGlobalDataIndex));
if (blacklisted)
QTestResult::setBlacklistCurrentTest(true);
}
QTestDataSetter s(curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex));
if (blacklisted && skipBlacklisted) {
QTest::qSkip("Skipping blacklisted test since -skipblacklisted option is set.",
NULL, 0);
QTestResult::finishedCurrentTestData();
QTestResult::finishedCurrentTestDataCleanup();
} else {
QTestDataSetter s(
curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex));
QTestPrivate::qtestMouseButtons = Qt::NoButton;
if (watchDog)
watchDog->beginTest();
QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call
invokeTestOnData(index);
if (watchDog)
watchDog->testFinished();
QTestPrivate::qtestMouseButtons = Qt::NoButton;
if (watchDog)
watchDog->beginTest();
QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call
invokeTestOnData(index);
if (watchDog)
watchDog->testFinished();
}
if (!tag.isEmpty() && !globalDataCount)
break;

1
tests/auto/testlib/selftests/CMakeLists.txt
View File

@ -95,6 +95,7 @@ set(subprograms
signaldumper
singleskip
skip
skipblacklisted
skipcleanup
skipcleanuptestcase
skipinit

18
tests/auto/testlib/selftests/expected_skipblacklisted.junitxml Normal file
View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_SkipBlacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="0" errors="0" skipped="3" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtBuild" value=""/>
</properties>
<testcase name="initTestCase" classname="tst_SkipBlacklisted" time="@TEST_DURATION@"/>
<testcase name="pass" classname="tst_SkipBlacklisted" time="@TEST_DURATION@"/>
<testcase name="blacklisted" classname="tst_SkipBlacklisted" time="@TEST_DURATION@">
<skipped message="Skipping blacklisted test since &#x002D;skipblacklisted option is set."/>
</testcase>
<testcase name="blacklistedData(should pass)" classname="tst_SkipBlacklisted" time="@TEST_DURATION@">
<skipped message="Skipping blacklisted test since &#x002D;skipblacklisted option is set."/>
<skipped message="Skipping blacklisted test since &#x002D;skipblacklisted option is set."/>
</testcase>
<testcase name="cleanupTestCase" classname="tst_SkipBlacklisted" time="@TEST_DURATION@"/>
</testsuite>

36
tests/auto/testlib/selftests/expected_skipblacklisted.lightxml Normal file
View File

@ -0,0 +1,36 @@
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="pass">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="blacklisted">
<Incident type="skip" file="" line="0">
<Description><![CDATA[Skipping blacklisted test since -skipblacklisted option is set.]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="blacklistedData">
<Incident type="skip" file="" line="0">
<Description><![CDATA[Skipping blacklisted test since -skipblacklisted option is set.]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[should pass]]></DataTag>
</Incident>
<Incident type="skip" file="" line="0">
<Description><![CDATA[Skipping blacklisted test since -skipblacklisted option is set.]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>

13
tests/auto/testlib/selftests/expected_skipblacklisted.tap Normal file
View File

@ -0,0 +1,13 @@
TAP version 13
# tst_SkipBlacklisted
ok 1 - initTestCase()
ok 2 - pass()
ok 3 - blacklisted() # SKIP Skipping blacklisted test since -skipblacklisted option is set.
ok 4 - blacklistedData() # SKIP Skipping blacklisted test since -skipblacklisted option is set.
ok 5 - blacklistedData(should pass)
ok 6 - blacklistedData() # SKIP Skipping blacklisted test since -skipblacklisted option is set.
ok 7 - cleanupTestCase()
1..7
# tests 7
# pass 4
# fail 0

19
tests/auto/testlib/selftests/expected_skipblacklisted.teamcity Normal file
View File

@ -0,0 +1,19 @@
##teamcity[testSuiteStarted name='tst_SkipBlacklisted' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='initTestCase()' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='initTestCase()' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='pass()' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='pass()' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='blacklisted()' flowId='tst_SkipBlacklisted']
##teamcity[testIgnored name='blacklisted()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='blacklisted()' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='blacklistedData()' flowId='tst_SkipBlacklisted']
##teamcity[testIgnored name='blacklistedData()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='blacklistedData()' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='blacklistedData(should pass)' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='blacklistedData(should pass)' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='blacklistedData()' flowId='tst_SkipBlacklisted']
##teamcity[testIgnored name='blacklistedData()' message='Skipping blacklisted test since -skipblacklisted option is set.' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='blacklistedData()' flowId='tst_SkipBlacklisted']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipBlacklisted']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipBlacklisted']
##teamcity[testSuiteFinished name='tst_SkipBlacklisted' flowId='tst_SkipBlacklisted']

11
tests/auto/testlib/selftests/expected_skipblacklisted.txt Normal file
View File

@ -0,0 +1,11 @@
********* Start testing of tst_SkipBlacklisted *********
Config: Using QtTest library
PASS : tst_SkipBlacklisted::initTestCase()
PASS : tst_SkipBlacklisted::pass()
SKIP : tst_SkipBlacklisted::blacklisted() Skipping blacklisted test since -skipblacklisted option is set.
SKIP : tst_SkipBlacklisted::blacklistedData() Skipping blacklisted test since -skipblacklisted option is set.
PASS : tst_SkipBlacklisted::blacklistedData(should pass)
SKIP : tst_SkipBlacklisted::blacklistedData() Skipping blacklisted test since -skipblacklisted option is set.
PASS : tst_SkipBlacklisted::cleanupTestCase()
Totals: 4 passed, 0 failed, 3 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_SkipBlacklisted *********

39
tests/auto/testlib/selftests/expected_skipblacklisted.xml Normal file
View File

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<TestCase name="tst_SkipBlacklisted">
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="pass">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="blacklisted">
<Incident type="skip" file="" line="0">
<Description><![CDATA[Skipping blacklisted test since -skipblacklisted option is set.]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="blacklistedData">
<Incident type="skip" file="" line="0">
<Description><![CDATA[Skipping blacklisted test since -skipblacklisted option is set.]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[should pass]]></DataTag>
</Incident>
<Incident type="skip" file="" line="0">
<Description><![CDATA[Skipping blacklisted test since -skipblacklisted option is set.]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>
</TestCase>

9
tests/auto/testlib/selftests/generate_expected_output.py
View File

@ -37,10 +37,11 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring',
'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',
'printdatatags', 'printdatatagswithglobaltags', 'qexecstringlist',
'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip', 'skipcleanup',
'skipcleanuptestcase', 'skipinit', 'skipinitdata', 'sleep', 'strcmp',
'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2',
'verifyexceptionthrown', 'warnings', 'watchdog', 'junit', 'keyboard']
'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip',
'skipblacklisted', 'skipcleanup', 'skipcleanuptestcase', 'skipinit',
'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', 'tuplediagnostics',
'verbose1', 'verbose2', 'verifyexceptionthrown', 'warnings', 'watchdog',
'junit', 'keyboard']
class Fail (Exception): pass

6
tests/auto/testlib/selftests/skipblacklisted/BLACKLIST Normal file
View File

@ -0,0 +1,6 @@
[blacklisted]
*
[blacklistedData:blacklisted 1]
*
[blacklistedData:blacklisted 2]
*

17
tests/auto/testlib/selftests/skipblacklisted/CMakeLists.txt Normal file
View File

@ -0,0 +1,17 @@
#####################################################################
## skipblacklisted Binary:
#####################################################################
qt_internal_add_executable(skipblacklisted
NO_INSTALL
OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
SOURCES
tst_skipblacklisted.cpp
LIBRARIES
Qt::Test
)
## Scopes:
#####################################################################
qt_internal_apply_testlib_coverage_options(skipblacklisted)

51
tests/auto/testlib/selftests/skipblacklisted/tst_skipblacklisted.cpp Normal file
View File

@ -0,0 +1,51 @@
// Copyright (C) 2023 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include <QTest>
class tst_SkipBlacklisted : public QObject
{
Q_OBJECT
private slots:
void pass();
void blacklisted();
void blacklistedData();
void blacklistedData_data();
};
void tst_SkipBlacklisted::pass()
{
QVERIFY(true);
}
// This test have been blacklisted in skipblacklisted/BLACKLIST
void tst_SkipBlacklisted::blacklisted()
{
QFAIL("this line should never be reached, since we skip all blacklisted test functions");
}
// blacklisted 1 and blacklisted 2 have been blacklisted in skipblacklisted/BLACKLIST
void tst_SkipBlacklisted::blacklistedData()
{
QFETCH(int, testdata);
QCOMPARE(testdata, 2);
}
void tst_SkipBlacklisted::blacklistedData_data()
{
QTest::addColumn<int>("testdata");
QTest::newRow("blacklisted 1") << 1;
QTest::newRow("should pass") << 2;
QTest::newRow("blacklisted 2") << 3;
}
QTEST_MAIN_WRAPPER(tst_SkipBlacklisted,
std::vector<const char*> args(argv, argv + argc);
args.push_back("-skipblacklisted");
argc = int(args.size());
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_SETUP())
#include "tst_skipblacklisted.moc"