AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

testlib: Clear expected failures after every data row.

Browse Source 
Previously, expected failures were only cleared at the end of each test
function, i.e. after all data rows were finished.  This meant that if a
data-driven test function called QEXPECT_FAIL and then didn't perform
any further verification steps to trigger the expected failure, the
expected failure would be carried over to the next data row, probably
causing the first verification step in the test function to XPASS (with
a seemingly irrelevant error message) for the next data row.

This commit adds the new function QTestResult::finishedCurrentTestData()
to cleanup after each data row is executed.  This function treats calls
to QEXPECT_FAIL without subsequent verification steps as a test failure.

This commit also adds a regression test to demonstrate that expected
failures can no longer be carried over from one data row to another. If
run against the previous version of testlib, the new test would report a
pass instead of an error.

Change-Id: Ida5c7f080815b0dca9531131fed582b0918334cb
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
This commit is contained in:
Jason McDonald 2012-02-01 15:42:00 +10:00 committed by Qt by Nokia
parent 684a1559f0
commit da18ada6bf
8 changed files with 87 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/testlib/qtestcase.cpp
View File

@ -1506,6 +1506,8 @@ static void qInvokeTestMethodDataEntry(char *slot)
if (!invokeOk)
QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__);
QTestResult::finishedCurrentTestData();
QTestResult::setCurrentTestLocation(QTestResult::CleanupFunc);
invokeMethod(QTest::currentTestObject, "cleanup()");
QTestResult::setCurrentTestLocation(QTestResult::NoWhere);

9
src/testlib/qtestresult.cpp
View File

@ -127,6 +127,13 @@ static void clearExpectFail()
QTest::expectFailComment = 0;
}
void QTestResult::finishedCurrentTestData()
{
if (QTest::expectFailMode)
addFailure("QEXPECT_FAIL was called without any subsequent verification statements", 0, 0);
clearExpectFail();
}
void QTestResult::finishedCurrentTestFunction()
{
if (!QTest::failed && QTestLog::unhandledIgnoreMessages()) {
@ -143,8 +150,6 @@ void QTestResult::finishedCurrentTestFunction()
QTest::location = NoWhere;
QTestLog::leaveTestFunction();
clearExpectFail();
}
const char *QTestResult::currentTestFunction()

1
src/testlib/qtestresult_p.h
View File

@ -73,6 +73,7 @@ public:
static TestLocation currentTestLocation();
static const char *currentDataTag();
static const char *currentGlobalDataTag();
static void finishedCurrentTestData();
static void finishedCurrentTestFunction();
static void reset();

32
tests/auto/testlib/selftests/expected_expectfail.lightxml
View File

@ -9,7 +9,7 @@
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="72">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="74">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
@ -21,31 +21,31 @@
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="80">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="82">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailTwice">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="90">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="92">
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
</TestFunction>
<TestFunction name="xfailWithQString">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="99">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="101">
<Description><![CDATA[A string]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="104">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="106">
<Description><![CDATA[Bug 5 (The message)]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailDataDriven">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="135">
<DataTag><![CDATA[Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="135">
<DataTag><![CDATA[Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
@ -55,23 +55,33 @@
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailOnAnyRow">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="170">
<DataTag><![CDATA[first row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="170">
<DataTag><![CDATA[second row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailWithoutVerify">
<Incident type="fail" file="" line="0">
<DataTag><![CDATA[first row]]></DataTag>
<Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description>
</Incident>
<Incident type="fail" file="" line="0">
<DataTag><![CDATA[second row]]></DataTag>
<Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description>
</Incident>
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="174">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="190">
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>
</TestFunction>
<TestFunction name="xpassDataDriven">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="196">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="212">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>

26
tests/auto/testlib/selftests/expected_expectfail.txt
View File

@ -3,35 +3,37 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE
PASS : tst_ExpectFail::initTestCase()
QDEBUG : tst_ExpectFail::xfailAndContinue() begin
XFAIL : tst_ExpectFail::xfailAndContinue() This should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(72)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(74)]
QDEBUG : tst_ExpectFail::xfailAndContinue() after
PASS : tst_ExpectFail::xfailAndContinue()
QDEBUG : tst_ExpectFail::xfailAndAbort() begin
XFAIL : tst_ExpectFail::xfailAndAbort() This should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(80)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(82)]
PASS : tst_ExpectFail::xfailAndAbort()
FAIL! : tst_ExpectFail::xfailTwice() Already expecting a fail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(90)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(92)]
XFAIL : tst_ExpectFail::xfailWithQString() A string
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(99)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(101)]
XFAIL : tst_ExpectFail::xfailWithQString() Bug 5 (The message)
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(104)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(106)]
PASS : tst_ExpectFail::xfailWithQString()
XFAIL : tst_ExpectFail::xfailDataDriven(Abort) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(133)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(135)]
XFAIL : tst_ExpectFail::xfailDataDriven(Continue) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(133)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(135)]
PASS : tst_ExpectFail::xfailDataDriven()
PASS : tst_ExpectFail::xfailOnWrongRow()
XFAIL : tst_ExpectFail::xfailOnAnyRow(first row) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(168)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(170)]
XFAIL : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(168)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(170)]
PASS : tst_ExpectFail::xfailOnAnyRow()
FAIL! : tst_ExpectFail::xfailWithoutVerify(first row) QEXPECT_FAIL was called without any subsequent verification statements
FAIL! : tst_ExpectFail::xfailWithoutVerify(second row) QEXPECT_FAIL was called without any subsequent verification statements
XPASS : tst_ExpectFail::xpass() 'true' returned FALSE. ()
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(172)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(190)]
XPASS : tst_ExpectFail::xpassDataDriven(XPass) 'true' returned FALSE. ()
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(196)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(212)]
PASS : tst_ExpectFail::cleanupTestCase()
Totals: 8 passed, 3 failed, 0 skipped
Totals: 8 passed, 5 failed, 0 skipped
********* Finished testing of tst_ExpectFail *********

32
tests/auto/testlib/selftests/expected_expectfail.xml
View File

@ -11,7 +11,7 @@
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="72">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="74">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
@ -23,31 +23,31 @@
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="80">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="82">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailTwice">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="90">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="92">
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
</TestFunction>
<TestFunction name="xfailWithQString">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="99">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="101">
<Description><![CDATA[A string]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="104">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="106">
<Description><![CDATA[Bug 5 (The message)]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailDataDriven">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="135">
<DataTag><![CDATA[Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="135">
<DataTag><![CDATA[Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
@ -57,23 +57,33 @@
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailOnAnyRow">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="170">
<DataTag><![CDATA[first row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="170">
<DataTag><![CDATA[second row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xfailWithoutVerify">
<Incident type="fail" file="" line="0">
<DataTag><![CDATA[first row]]></DataTag>
<Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description>
</Incident>
<Incident type="fail" file="" line="0">
<DataTag><![CDATA[second row]]></DataTag>
<Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description>
</Incident>
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="174">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="190">
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>
</TestFunction>
<TestFunction name="xpassDataDriven">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="196">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="212">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>

6
tests/auto/testlib/selftests/expected_expectfail.xunitxml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite errors="11" failures="3" tests="11" name="tst_ExpectFail">
<testsuite errors="11" failures="5" tests="12" name="tst_ExpectFail">
<properties>
<property value="5.0.0" name="QTestVersion"/>
<property value="5.0.0" name="QtVersion"/>
@ -30,6 +30,10 @@
<!-- tag="first row" message="This test should xfail" type="info" -->
<!-- tag="second row" message="This test should xfail" type="info" -->
</testcase>
<testcase result="fail" name="xfailWithoutVerify">
<failure tag="first row" message="QEXPECT_FAIL was called without any subsequent verification statements" result="fail"/>
<failure tag="second row" message="QEXPECT_FAIL was called without any subsequent verification statements" result="fail"/>
</testcase>
<testcase result="xpass" name="xpass">
<failure message="&apos;true&apos; returned FALSE. ()" result="xpass"/>
</testcase>

16
tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
View File

@ -60,6 +60,8 @@ private slots:
void xfailOnWrongRow() const;
void xfailOnAnyRow_data() const;
void xfailOnAnyRow() const;
void xfailWithoutVerify_data() const;
void xfailWithoutVerify() const;
void xpass() const;
void xpassDataDriven_data() const;
void xpassDataDriven() const;
@ -168,6 +170,20 @@ void tst_ExpectFail::xfailOnAnyRow() const
QVERIFY(false);
}
void tst_ExpectFail::xfailWithoutVerify_data() const
{
QTest::addColumn<int>("dummy");
QTest::newRow("first row") << 0;
QTest::newRow("second row") << 1;
}
void tst_ExpectFail::xfailWithoutVerify() const
{
QVERIFY(true);
QEXPECT_FAIL("", "This expected failure should be ignored", Abort);
}
void tst_ExpectFail::xpass() const
{
QEXPECT_FAIL("", "This test should xpass", Abort);