AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
77a93e6df3
qt5base-lts/tests/auto/testlib/selftests/expected_expectfail.xml
Edward Welbourne 77a93e6df3 Make Skip an incident in test logging 
Skip ends the test (albeit inconclusively).  Rearrange the enums in
the abstract logger, move code to handle skip between relevant
function and tidy up various things that became simpler as a result.

Also reorder the message enum, and its switches, to separate testlib's
internals from the usual Qt messages, and put each group in ascending
order of severity.

Task-number: QTBUG-96844
Change-Id: I2c7a634b9f849830d64eafa750155e66e244b729
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
2021-12-09 20:54:26 +01:00

325 lines
14 KiB
XML
Raw Blame History

<?xml version="1.0" encoding="UTF-8"?>
<TestCase name="tst_ExpectFail">
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailAndContinue">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[after]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailAndAbort">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailContinueSkip">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This should xfail then skip]]></Description>
</Incident>
<Incident type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This skip should be reported and counted]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailAbortSkip">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailTwice">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailDataDrivenTwice">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Pass Abort]]></DataTag>
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Pass Continue]]></DataTag>
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailWithQString">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[A string]]></Description>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[Bug 5 (The message)]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailDataDrivenWithQString">
<Incident type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Pass Abort]]></DataTag>
<Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description>
</Incident>
<Incident type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Pass Continue]]></DataTag>
<Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[A string]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[A string]]></Description>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[Bug 5 (The message)]]></Description>
</Incident>
<Incident type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailDataDrivenWithQVerify">
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass Abort]]></DataTag>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass Continue]]></DataTag>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailDataDrivenWithQCompare">
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass Abort]]></DataTag>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass Continue]]></DataTag>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailOnWrongRow">
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[Should pass (*not* xpass), despite test-case name]]></Description>
</Message>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
</Incident>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[Should pass (*not* xpass), despite test-case name]]></Description>
</Message>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailOnAnyRow">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xfailWithoutCheck">
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[Should fail (*not* xfail), despite test-case name]]></Description>
</Message>
<Incident type="fail" file="" line="0">
<DataTag><![CDATA[Fail Abort]]></DataTag>
<Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[Should fail (*not* xfail), despite test-case name]]></Description>
</Message>
<Incident type="fail" file="" line="0">
<DataTag><![CDATA[Fail Continue]]></DataTag>
<Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassAbort">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassAbortSkip">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassAbortXfailContinue">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassContinue">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[This should be reached]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassContinueSkip">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Incident type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This should be reached but not increment skip-count]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassContinueXfailAbort">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA[This test should xfail but not add to totals]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassAbortDataDrivenWithQVerify">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassContinueDataDrivenWithQVerify">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA[Test should Continue past XPASS]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[Pass]]></DataTag>
<Description><![CDATA[Test should simply PASS]]></Description>
</Message>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassAbortDataDrivenWithQCompare">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA[QCOMPARE(1, 1) returned TRUE unexpectedly.]]></Description>
</Incident>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="xpassContinueDataDrivenWithQCompare">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA[QCOMPARE(1, 1) returned TRUE unexpectedly.]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[XPass]]></DataTag>
<Description><![CDATA[Test should Continue past XPASS]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[Pass]]></DataTag>
<Description><![CDATA[Test should simply PASS]]></Description>
</Message>
<Incident type="pass" file="" line="0">
<DataTag><![CDATA[Pass]]></DataTag>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>
</TestCase>
Reference in New Issue View Git Blame Copy Permalink