AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

testlib: Add BXPASS and BXFAIL

Browse Source 
Prioritize blacklisting over QEXPECT_FAIL so that a test that is
blacklisted no longer fails if QEXPECT_FAIL returns true unexpectedly. To
reflect this state properly, the two values of BXPASS and BXFAIL were
added to testlib's output.

[ChangeLog][Important Behavior Changes][QtTestLib] Blacklisting of tests
will be taken into account for XPASS and XFAIL. A blacklisted test that
causes an XPASS will no longer be a fail.

Task-number: QTBUG-72928
Change-Id: Ia2232fdc714d405fa3fd9aea6c89eb2836bc5950
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Oliver Wolff 2019-01-15 09:58:34 +01:00 committed by Tony Sarajärvi
parent 3615e8aaa1
commit 9906cc57ed
16 changed files with 81 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/testlib/qabstracttestlogger_p.h
View File

@ -69,7 +69,9 @@ public:
Fail,
XPass,
BlacklistedPass,
BlacklistedFail
BlacklistedFail,
BlacklistedXPass,
BlacklistedXFail
};
enum MessageTypes {

4
src/testlib/qappletestlogger.cpp
View File

@ -101,6 +101,10 @@ static IncidentClassification incidentTypeToClassification(QAbstractTestLogger::
return IncidentClassification(QtWarningMsg, "bpass");
case QAbstractTestLogger::BlacklistedFail:
return IncidentClassification(QtInfoMsg, "bfail");
case QAbstractTestLogger::BlacklistedXPass:
return IncidentClassification(QtWarningMsg, "bxpass");
case QAbstractTestLogger::BlacklistedXFail:
return IncidentClassification(QtInfoMsg, "bxfail");
}
return IncidentClassification(QtFatalMsg, nullptr);
}

4
src/testlib/qplaintestlogger.cpp
View File

@ -89,6 +89,10 @@ namespace QTest {
return "BPASS ";
case QAbstractTestLogger::BlacklistedFail:
return "BFAIL ";
case QAbstractTestLogger::BlacklistedXPass:
return "BXPASS ";
case QAbstractTestLogger::BlacklistedXFail:
return "BXFAIL ";
}
return "??????";
}

6
src/testlib/qtaptestlogger.cpp
View File

@ -123,13 +123,15 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
return;
}
bool ok = type == Pass || type == XPass || type == BlacklistedPass;
bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass;
QTestCharBuffer directive;
if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass)
if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass
|| type == BlacklistedXFail || type == BlacklistedXPass) {
// We treat expected or blacklisted failures/passes as TODO-failures/passes,
// which should be treated as soft issues by consumers. Not all do though :/
QTest::qt_asprintf(&directive, " # TODO %s", description);
}
int testNumber = QTestLog::totalCount();
if (type == XFail) {

4
src/testlib/qteamcitylogger.cpp
View File

@ -66,6 +66,10 @@ namespace QTest {
return "BPASS";
case QAbstractTestLogger::BlacklistedFail:
return "BFAIL";
case QAbstractTestLogger::BlacklistedXPass:
return "BXPASS";
case QAbstractTestLogger::BlacklistedXFail:
return "BXFAIL";
}
return "??????";
}

18
src/testlib/qtestlog.cpp
View File

@ -455,6 +455,24 @@ void QTestLog::addBFail(const char *msg, const char *file, int line)
QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
}
void QTestLog::addBXPass(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
QTEST_ASSERT(file);
++QTest::blacklists;
QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
}
void QTestLog::addBXFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
QTEST_ASSERT(file);
QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
}
void QTestLog::addSkip(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);

2
src/testlib/qtestlog_p.h
View File

@ -80,6 +80,8 @@ public:
static void addXPass(const char *msg, const char *file, int line);
static void addBPass(const char *msg);
static void addBFail(const char *msg, const char *file, int line);
static void addBXPass(const char *msg, const char *file, int line);
static void addBXFail(const char *msg, const char *file, int line);
static void addSkip(const char *msg, const char *file, int line);
static void addBenchmarkResult(const QBenchmarkResult &result);

13
src/testlib/qtestresult.cpp
View File

@ -218,17 +218,24 @@ static bool checkStatement(bool statement, const char *msg, const char *file, in
{
if (statement) {
if (QTest::expectFailMode) {
QTestLog::addXPass(msg, file, line);
if (QTest::blacklistCurrentTest)
QTestLog::addBXPass(msg, file, line);
else
QTestLog::addXPass(msg, file, line);
QTest::failed = true;
bool doContinue = (QTest::expectFailMode == QTest::Continue);
clearExpectFail();
QTest::failed = true;
return doContinue;
}
return true;
}
if (QTest::expectFailMode) {
QTestLog::addXFail(QTest::expectFailComment, file, line);
if (QTest::blacklistCurrentTest)
QTestLog::addBXFail(QTest::expectFailComment, file, line);
else
QTestLog::addXFail(QTest::expectFailComment, file, line);
bool doContinue = (QTest::expectFailMode == QTest::Continue);
clearExpectFail();
return doContinue;

4
src/testlib/qxmltestlogger.cpp
View File

@ -91,6 +91,10 @@ namespace QTest {
return "bpass";
case QAbstractTestLogger::BlacklistedFail:
return "bfail";
case QAbstractTestLogger::BlacklistedXPass:
return "bxpass";
case QAbstractTestLogger::BlacklistedXFail:
return "bxfail";
}
return "??????";
}

13
src/testlib/qxunittestlogger.cpp
View File

@ -180,6 +180,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
++failureCounter;
typeBuf = "bfail";
break;
case QAbstractTestLogger::BlacklistedXPass:
typeBuf = "bxpass";
break;
case QAbstractTestLogger::BlacklistedXFail:
++failureCounter;
typeBuf = "bxfail";
break;
default:
typeBuf = "??????";
break;
@ -212,11 +219,11 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
if (!strcmp(oldResult, "pass")) {
overwrite = true;
}
else if (!strcmp(oldResult, "bpass")) {
else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
|| (type == QAbstractTestLogger::BlacklistedFail);
|| (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
}
else if (!strcmp(oldResult, "bfail")) {
else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
}
else if (!strcmp(oldResult, "xfail")) {

6
tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
View File

@ -64,14 +64,14 @@ void tst_Blacklisted::fail()
void tst_Blacklisted::xfail()
{
QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort);
QEXPECT_FAIL("", "This test should BXFAIL then BPASS", Abort);
QVERIFY(false);
}
void tst_Blacklisted::xpass()
{
QEXPECT_FAIL("", "This test should XPASS", Abort);
QVERIFY2(true, "This test should XPASS, blacklist ignored for XPASS");
QEXPECT_FAIL("", "This test should BXPASS", Abort);
QVERIFY2(true, "This test should BXPASS");
}
void tst_Blacklisted::messages()

8
tests/auto/testlib/selftests/expected_blacklisted.lightxml
View File

@ -24,15 +24,15 @@
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfail">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA[This test should BXFAIL then BPASS]]></Description>
</Incident>
<Incident type="bpass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)]]></Description>
<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>

3
tests/auto/testlib/selftests/expected_blacklisted.teamcity
View File

@ -7,10 +7,9 @@
##teamcity[testStarted name='fail()' flowId='tst_Blacklisted']
##teamcity[testFinished name='fail()' flowId='tst_Blacklisted']
##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted']
##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted']
##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted']
##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted']
##teamcity[testStarted name='messages()' flowId='tst_Blacklisted']
##teamcity[testStdOut name='messages()' out='QWARN: This is a warning that should not appear in silent test output|nWARNING |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib warning that should not appear in silent test output|nQDEBUG: This is a debug message that should not appear in silent test output|nQSYSTEM: This is a critical message that should not appear in silent test output|nQINFO: This is an info message that should not appear in silent test output|nINFO |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib info message that should not appear in silent test output|nQFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Blacklisted']

6
tests/auto/testlib/selftests/expected_blacklisted.txt
View File

@ -6,10 +6,10 @@ SKIP : tst_Blacklisted::skip() This test should SKIP
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL)
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BFAIL
BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
BPASS : tst_Blacklisted::xfail()
XPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS)
Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
QWARN : tst_Blacklisted::messages() This is a warning that should not appear in silent test output
WARNING: tst_Blacklisted::messages() This is an internal testlib warning that should not appear in silent test output
@ -22,5 +22,5 @@ INFO : tst_Blacklisted::messages() This is an internal testlib info message th
QFATAL : tst_Blacklisted::messages() This is a fatal error message that should still appear in silent test output
BFAIL : tst_Blacklisted::messages() Received a fatal error.
Loc: [Unknown file(0)]
Totals: 1 passed, 1 failed, 1 skipped, 4 blacklisted, 0ms
Totals: 1 passed, 0 failed, 1 skipped, 5 blacklisted, 0ms
********* Finished testing of tst_Blacklisted *********

8
tests/auto/testlib/selftests/expected_blacklisted.xml
View File

@ -26,15 +26,15 @@
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfail">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA[This test should BXFAIL then BPASS]]></Description>
</Incident>
<Incident type="bpass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)]]></Description>
<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>

11
tests/auto/testlib/selftests/expected_blacklisted.xunitxml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite errors="9" failures="3" tests="7" name="tst_Blacklisted">
<testsuite errors="8" failures="3" tests="7" name="tst_Blacklisted">
<properties>
<property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
<property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@ -11,12 +11,8 @@
<!-- message="This test should SKIP" type="skip" -->
</testcase>
<testcase result="bfail" name="fail"/>
<testcase result="xfail" name="xfail">
<!-- message="This test should XFAIL then BFAIL" type="info" -->
</testcase>
<testcase result="xpass" name="xpass">
<failure message="&apos;true&apos; returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/>
</testcase>
<testcase result="bxfail" name="xfail"/>
<testcase result="bxpass" name="xpass"/>
<testcase result="bfail" name="messages">
<!-- message="This is a warning that should not appear in silent test output" type="qwarn" -->
<!-- message="This is an internal testlib warning that should not appear in silent test output" type="warn" -->
@ -28,7 +24,6 @@
</testcase>
<system-err>
<![CDATA[This test should SKIP]]>
<![CDATA[This test should XFAIL then BFAIL]]>
<![CDATA[This is a warning that should not appear in silent test output]]>
<![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
<![CDATA[This is a debug message that should not appear in silent test output]]>