AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
21e9c223b7
qt5base-lts/tests/auto/testlib/selftests/expected_expectfail.junitxml
Edward Welbourne 13d2e13290 Make counts of various types of test result add up correctly 
Added tests for repeated skips and failures (from within void lambdas,
to simulate skips and failures from within event handlers). These
exhibit yet more ways to count more than one outcome for a test.  The
new QTest::failOnWarning() can also provoke more than one failure from
a single test, and several existing selftests exhibited various ways
for the Totals line's counts to add up to more than the number of
actual tests run.

Fixed counting so that only the first decisive incident is counted.
Tests can still report later failure or skipping, but only the first
is counted.

Added a currentTestState in qtestlog.cpp, by which it keeps track of
whether the test has resolved to a result, and clearCurrentTestState()
by which other code can reset that at the end of each test. This
brought to light various places where test-end clean-up was not being
handled - due to failure or skipping in a *_data() method or init, or
a skip in cleanup.

Fixes: QTBUG-95661
Change-Id: I5d24a37a53d3db225fa602649d8aad8f5ed6c1ad
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
2022-01-13 19:47:22 +01:00

189 lines
9.1 KiB
XML
Raw Blame History

<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="4" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtBuild" value=""/>
</properties>
<testcase name="initTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xfailAndContinue" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[begin]]>
<![CDATA[This should xfail]]>
<![CDATA[after]]>
</system-out>
</testcase>
<testcase name="xfailAndAbort" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[begin]]>
<![CDATA[This should xfail]]>
</system-out>
</testcase>
<testcase name="xfailContinueSkip" classname="tst_ExpectFail" time="@TEST_DURATION@">
<skipped message="This skip should be reported and counted"/>
<system-out>
<![CDATA[This should xfail then skip]]>
</system-out>
</testcase>
<testcase name="xfailAbortSkip" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This should xfail]]>
</system-out>
</testcase>
<testcase name="xfailTwice" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="Already expecting a fail"/>
</testcase>
<testcase name="xfailDataDrivenTwice(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="Already expecting a fail"/>
</testcase>
<testcase name="xfailDataDrivenTwice(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="Already expecting a fail"/>
</testcase>
<testcase name="xfailDataDrivenTwice(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="Already expecting a fail"/>
</testcase>
<testcase name="xfailDataDrivenTwice(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="Already expecting a fail"/>
</testcase>
<testcase name="xfailWithQString" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[A string]]>
<![CDATA[Bug 5 (The message)]]>
</system-out>
</testcase>
<testcase name="xfailDataDrivenWithQString(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<skipped message="Each Continue or Pass reports this and increments skip&#x002D;count"/>
</testcase>
<testcase name="xfailDataDrivenWithQString(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<skipped message="Each Continue or Pass reports this and increments skip&#x002D;count"/>
</testcase>
<testcase name="xfailDataDrivenWithQString(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[A string]]>
</system-out>
</testcase>
<testcase name="xfailDataDrivenWithQString(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<skipped message="Each Continue or Pass reports this and increments skip&#x002D;count"/>
<system-out>
<![CDATA[A string]]>
<![CDATA[Bug 5 (The message)]]>
</system-out>
</testcase>
<testcase name="xfailDataDrivenWithQVerify(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xfailDataDrivenWithQVerify(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xfailDataDrivenWithQVerify(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This test should xfail]]>
</system-out>
</testcase>
<testcase name="xfailDataDrivenWithQVerify(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This test should xfail]]>
</system-out>
</testcase>
<testcase name="xfailDataDrivenWithQCompare(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xfailDataDrivenWithQCompare(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xfailDataDrivenWithQCompare(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This test should xfail]]>
</system-out>
</testcase>
<testcase name="xfailDataDrivenWithQCompare(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This test should xfail]]>
</system-out>
</testcase>
<testcase name="xfailOnWrongRow(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[Should pass (*not* xpass), despite test-case name]]>
</system-out>
</testcase>
<testcase name="xfailOnWrongRow(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[Should pass (*not* xpass), despite test-case name]]>
</system-out>
</testcase>
<testcase name="xfailOnAnyRow(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This test should xfail]]>
</system-out>
</testcase>
<testcase name="xfailOnAnyRow(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[This test should xfail]]>
</system-out>
</testcase>
<testcase name="xfailWithoutCheck(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="QEXPECT_FAIL was called without any subsequent verification statements"/>
<system-out>
<![CDATA[Should fail (*not* xfail), despite test-case name]]>
</system-out>
</testcase>
<testcase name="xfailWithoutCheck(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="QEXPECT_FAIL was called without any subsequent verification statements"/>
<system-out>
<![CDATA[Should fail (*not* xfail), despite test-case name]]>
</system-out>
</testcase>
<testcase name="xpassAbort" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
</testcase>
<testcase name="xpassAbortSkip" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
</testcase>
<testcase name="xpassAbortXfailContinue" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
</testcase>
<testcase name="xpassContinue" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
<system-out>
<![CDATA[This should be reached]]>
</system-out>
</testcase>
<testcase name="xpassContinueSkip" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
<skipped message="This should be reached but not increment skip&#x002D;count"/>
</testcase>
<testcase name="xpassContinueXfailAbort" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
<system-out>
<![CDATA[This test should xfail but not add to totals]]>
</system-out>
</testcase>
<testcase name="xpassAbortDataDrivenWithQVerify(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
</testcase>
<testcase name="xpassAbortDataDrivenWithQVerify(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xpassContinueDataDrivenWithQVerify(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
<system-out>
<![CDATA[Test should Continue past XPASS]]>
</system-out>
</testcase>
<testcase name="xpassContinueDataDrivenWithQVerify(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[Test should simply PASS]]>
</system-out>
</testcase>
<testcase name="xpassAbortDataDrivenWithQCompare(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="QCOMPARE(1, 1) returned TRUE unexpectedly."/>
</testcase>
<testcase name="xpassAbortDataDrivenWithQCompare(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
<testcase name="xpassContinueDataDrivenWithQCompare(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="xpass" message="QCOMPARE(1, 1) returned TRUE unexpectedly."/>
<system-out>
<![CDATA[Test should Continue past XPASS]]>
</system-out>
</testcase>
<testcase name="xpassContinueDataDrivenWithQCompare(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[Test should simply PASS]]>
</system-out>
</testcase>
<testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@">
<system-out>
<![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]>
</system-out>
</testcase>
</testsuite>
Reference in New Issue View Git Blame Copy Permalink