AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a repeated-failure selftest for testlib

Browse Source 
This mimics the effects of test-code delegating checks to other
functions, which can lead to repeated failures. The failing test
should only contribute one to the total of failed tsts.  Drive-by:
correct an existing test's reporting of its own name.

Task-number: QTBUG-95661
Change-Id: I370fd2aee378d9fdd35826f0d11677483c60423d
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
This commit is contained in:
Edward Welbourne 2021-08-16 17:43:00 +02:00
parent 313de93b1d
commit bac150f056
7 changed files with 263 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tests/auto/testlib/selftests/expected_subtest.junitxml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?> <?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="9" failures="2" errors="0" skipped="0" time="@TEST_DURATION@"> <testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="10" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
<properties> <properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -48,8 +48,8 @@
<![CDATA[test3_data test3 (null)]]> <![CDATA[test3_data test3 (null)]]>
<![CDATA[test3_data end]]> <![CDATA[test3_data end]]>
<![CDATA[init test3 data0]]> <![CDATA[init test3 data0]]>
<![CDATA[test2 test3 data0]]> <![CDATA[test3 test3 data0]]>
<![CDATA[test2 end]]> <![CDATA[test3 end]]>
<![CDATA[cleanup test3 data0]]> <![CDATA[cleanup test3 data0]]>
</system-out> </system-out>
</testcase> </testcase>
@ -60,7 +60,7 @@
</failure> </failure>
<system-out> <system-out>
<![CDATA[init test3 data1]]> <![CDATA[init test3 data1]]>
<![CDATA[test2 test3 data1]]> <![CDATA[test3 test3 data1]]>
<![CDATA[cleanup test3 data1]]> <![CDATA[cleanup test3 data1]]>
</system-out> </system-out>
</testcase> </testcase>
@ -71,10 +71,17 @@
</failure> </failure>
<system-out> <system-out>
<![CDATA[init test3 data2]]> <![CDATA[init test3 data2]]>
<![CDATA[test2 test3 data2]]> <![CDATA[test3 test3 data2]]>
<![CDATA[cleanup test3 data2]]> <![CDATA[cleanup test3 data2]]>
</system-out> </system-out>
</testcase> </testcase>
<testcase name="multiFail" classname="tst_Subtest" time="@TEST_DURATION@">
<failure type="fail" message="This failure message should be repeated ten times"/>
<system-out>
<![CDATA[init multiFail (null)]]>
<![CDATA[cleanup multiFail (null)]]>
</system-out>
</testcase>
<testcase name="cleanupTestCase" classname="tst_Subtest" time="@TEST_DURATION@"> <testcase name="cleanupTestCase" classname="tst_Subtest" time="@TEST_DURATION@">
<system-out> <system-out>
<![CDATA[cleanupTestCase cleanupTestCase (null)]]> <![CDATA[cleanupTestCase cleanupTestCase (null)]]>

50
tests/auto/testlib/selftests/expected_subtest.lightxml
View File

@ -102,11 +102,11 @@
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data0]]></DataTag> <DataTag><![CDATA[data0]]></DataTag>
<Description><![CDATA[test2 test3 data0]]></Description> <Description><![CDATA[test3 test3 data0]]></Description>
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data0]]></DataTag> <DataTag><![CDATA[data0]]></DataTag>
<Description><![CDATA[test2 end]]></Description> <Description><![CDATA[test3 end]]></Description>
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data0]]></DataTag> <DataTag><![CDATA[data0]]></DataTag>
@ -121,7 +121,7 @@
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data1]]></DataTag> <DataTag><![CDATA[data1]]></DataTag>
<Description><![CDATA[test2 test3 data1]]></Description> <Description><![CDATA[test3 test3 data1]]></Description>
</Message> </Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<DataTag><![CDATA[data1]]></DataTag> <DataTag><![CDATA[data1]]></DataTag>
@ -139,7 +139,7 @@
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data2]]></DataTag> <DataTag><![CDATA[data2]]></DataTag>
<Description><![CDATA[test2 test3 data2]]></Description> <Description><![CDATA[test3 test3 data2]]></Description>
</Message> </Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<DataTag><![CDATA[data2]]></DataTag> <DataTag><![CDATA[data2]]></DataTag>
@ -153,6 +153,48 @@
</Message> </Message>
<Duration msecs="0"/> <Duration msecs="0"/>
</TestFunction> </TestFunction>
<TestFunction name="multiFail">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[init multiFail (null)]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[But this test should only contribute one to the failure count]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[cleanup multiFail (null)]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase"> <TestFunction name="cleanupTestCase">
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description> <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description>

95
tests/auto/testlib/selftests/expected_subtest.tap
View File

@ -26,12 +26,12 @@ ok 5 - test2(data2)
# test3_data test3 (null) # test3_data test3 (null)
# test3_data end # test3_data end
# init test3 data0 # init test3 data0
# test2 test3 data0 # test3 test3 data0
# test2 end # test3 end
# cleanup test3 data0 # cleanup test3 data0
ok 6 - test3(data0) ok 6 - test3(data0)
# init test3 data1 # init test3 data1
# test2 test3 data1 # test3 test3 data1
not ok 7 - test3(data1) not ok 7 - test3(data1)
--- ---
type: QCOMPARE type: QCOMPARE
@ -46,7 +46,7 @@ not ok 7 - test3(data1)
... ...
# cleanup test3 data1 # cleanup test3 data1
# init test3 data2 # init test3 data2
# test2 test3 data2 # test3 test3 data2
not ok 8 - test3(data2) not ok 8 - test3(data2)
--- ---
type: QCOMPARE type: QCOMPARE
@ -60,9 +60,88 @@ not ok 8 - test3(data2)
line: 0 line: 0
... ...
# cleanup test3 data2 # cleanup test3 data2
# init multiFail (null)
not ok 9 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 10 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 11 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 12 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 13 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 14 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 15 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 16 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 17 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 18 - multiFail()
---
# This failure message should be repeated ten times
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
not ok 19 - multiFail()
---
# But this test should only contribute one to the failure count
at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
line: 0
...
# cleanup multiFail (null)
# cleanupTestCase cleanupTestCase (null) # cleanupTestCase cleanupTestCase (null)
ok 9 - cleanupTestCase() ok 20 - cleanupTestCase()
1..9 1..20
# tests 9 # tests 20
# pass 7 # pass 7
# fail 2 # fail 13

32
tests/auto/testlib/selftests/expected_subtest.teamcity
View File

@ -15,17 +15,41 @@
##teamcity[testStdOut name='test2(data2)' out='QDEBUG: init test2 data2|nQDEBUG: test2 test2 data2|nQDEBUG: test2 end|nQDEBUG: cleanup test2 data2' flowId='tst_Subtest'] ##teamcity[testStdOut name='test2(data2)' out='QDEBUG: init test2 data2|nQDEBUG: test2 test2 data2|nQDEBUG: test2 end|nQDEBUG: cleanup test2 data2' flowId='tst_Subtest']
##teamcity[testFinished name='test2(data2)' flowId='tst_Subtest'] ##teamcity[testFinished name='test2(data2)' flowId='tst_Subtest']
##teamcity[testStarted name='test3(data0)' flowId='tst_Subtest'] ##teamcity[testStarted name='test3(data0)' flowId='tst_Subtest']
##teamcity[testStdOut name='test3(data0)' out='QDEBUG: test3_data test3 (null)|nQDEBUG: test3_data end|nQDEBUG: init test3 data0|nQDEBUG: test2 test3 data0|nQDEBUG: test2 end|nQDEBUG: cleanup test3 data0' flowId='tst_Subtest'] ##teamcity[testStdOut name='test3(data0)' out='QDEBUG: test3_data test3 (null)|nQDEBUG: test3_data end|nQDEBUG: init test3 data0|nQDEBUG: test3 test3 data0|nQDEBUG: test3 end|nQDEBUG: cleanup test3 data0' flowId='tst_Subtest']
##teamcity[testFinished name='test3(data0)' flowId='tst_Subtest'] ##teamcity[testFinished name='test3(data0)' flowId='tst_Subtest']
##teamcity[testStarted name='test3(data1)' flowId='tst_Subtest'] ##teamcity[testStarted name='test3(data1)' flowId='tst_Subtest']
##teamcity[testFailed name='test3(data1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello1"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest'] ##teamcity[testFailed name='test3(data1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello1"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest']
##teamcity[testStdOut name='test3(data1)' out='QDEBUG: init test3 data1|nQDEBUG: test2 test3 data1' flowId='tst_Subtest'] ##teamcity[testStdOut name='test3(data1)' out='QDEBUG: init test3 data1|nQDEBUG: test3 test3 data1' flowId='tst_Subtest']
##teamcity[testFinished name='test3(data1)' flowId='tst_Subtest'] ##teamcity[testFinished name='test3(data1)' flowId='tst_Subtest']
##teamcity[testStarted name='test3(data2)' flowId='tst_Subtest'] ##teamcity[testStarted name='test3(data2)' flowId='tst_Subtest']
##teamcity[testFailed name='test3(data2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello2"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest'] ##teamcity[testFailed name='test3(data2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello2"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest']
##teamcity[testStdOut name='test3(data2)' out='QDEBUG: cleanup test3 data1|nQDEBUG: init test3 data2|nQDEBUG: test2 test3 data2' flowId='tst_Subtest'] ##teamcity[testStdOut name='test3(data2)' out='QDEBUG: cleanup test3 data1|nQDEBUG: init test3 data2|nQDEBUG: test3 test3 data2' flowId='tst_Subtest']
##teamcity[testFinished name='test3(data2)' flowId='tst_Subtest'] ##teamcity[testFinished name='test3(data2)' flowId='tst_Subtest']
##teamcity[testStarted name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testStdOut name='multiFail()' out='QDEBUG: cleanup test3 data2|nQDEBUG: init multiFail (null)' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='But this test should only contribute one to the failure count' flowId='tst_Subtest']
##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest']
##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup test3 data2|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] ##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest']
##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest'] ##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest']

34
tests/auto/testlib/selftests/expected_subtest.txt
View File

@ -26,25 +26,49 @@ PASS : tst_Subtest::test2(data2)
QDEBUG : tst_Subtest::test3() test3_data test3 (null) QDEBUG : tst_Subtest::test3() test3_data test3 (null)
QDEBUG : tst_Subtest::test3() test3_data end QDEBUG : tst_Subtest::test3() test3_data end
QDEBUG : tst_Subtest::test3(data0) init test3 data0 QDEBUG : tst_Subtest::test3(data0) init test3 data0
QDEBUG : tst_Subtest::test3(data0) test2 test3 data0 QDEBUG : tst_Subtest::test3(data0) test3 test3 data0
QDEBUG : tst_Subtest::test3(data0) test2 end QDEBUG : tst_Subtest::test3(data0) test3 end
QDEBUG : tst_Subtest::test3(data0) cleanup test3 data0 QDEBUG : tst_Subtest::test3(data0) cleanup test3 data0
PASS : tst_Subtest::test3(data0) PASS : tst_Subtest::test3(data0)
QDEBUG : tst_Subtest::test3(data1) init test3 data1 QDEBUG : tst_Subtest::test3(data1) init test3 data1
QDEBUG : tst_Subtest::test3(data1) test2 test3 data1 QDEBUG : tst_Subtest::test3(data1) test3 test3 data1
FAIL! : tst_Subtest::test3(data1) Compared values are not the same FAIL! : tst_Subtest::test3(data1) Compared values are not the same
Actual (str) : "hello1" Actual (str) : "hello1"
Expected (QString("hello0")): "hello0" Expected (QString("hello0")): "hello0"
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
QDEBUG : tst_Subtest::test3(data1) cleanup test3 data1 QDEBUG : tst_Subtest::test3(data1) cleanup test3 data1
QDEBUG : tst_Subtest::test3(data2) init test3 data2 QDEBUG : tst_Subtest::test3(data2) init test3 data2
QDEBUG : tst_Subtest::test3(data2) test2 test3 data2 QDEBUG : tst_Subtest::test3(data2) test3 test3 data2
FAIL! : tst_Subtest::test3(data2) Compared values are not the same FAIL! : tst_Subtest::test3(data2) Compared values are not the same
Actual (str) : "hello2" Actual (str) : "hello2"
Expected (QString("hello0")): "hello0" Expected (QString("hello0")): "hello0"
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
QDEBUG : tst_Subtest::test3(data2) cleanup test3 data2 QDEBUG : tst_Subtest::test3(data2) cleanup test3 data2
QDEBUG : tst_Subtest::multiFail() init multiFail (null)
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
FAIL! : tst_Subtest::multiFail() But this test should only contribute one to the failure count
Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
QDEBUG : tst_Subtest::multiFail() cleanup multiFail (null)
QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null) QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null)
PASS : tst_Subtest::cleanupTestCase() PASS : tst_Subtest::cleanupTestCase()
Totals: 7 passed, 2 failed, 0 skipped, 0 blacklisted, 0ms Totals: 7 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_Subtest ********* ********* Finished testing of tst_Subtest *********

50
tests/auto/testlib/selftests/expected_subtest.xml
View File

@ -104,11 +104,11 @@
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data0]]></DataTag> <DataTag><![CDATA[data0]]></DataTag>
<Description><![CDATA[test2 test3 data0]]></Description> <Description><![CDATA[test3 test3 data0]]></Description>
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data0]]></DataTag> <DataTag><![CDATA[data0]]></DataTag>
<Description><![CDATA[test2 end]]></Description> <Description><![CDATA[test3 end]]></Description>
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data0]]></DataTag> <DataTag><![CDATA[data0]]></DataTag>
@ -123,7 +123,7 @@
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data1]]></DataTag> <DataTag><![CDATA[data1]]></DataTag>
<Description><![CDATA[test2 test3 data1]]></Description> <Description><![CDATA[test3 test3 data1]]></Description>
</Message> </Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<DataTag><![CDATA[data1]]></DataTag> <DataTag><![CDATA[data1]]></DataTag>
@ -141,7 +141,7 @@
</Message> </Message>
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<DataTag><![CDATA[data2]]></DataTag> <DataTag><![CDATA[data2]]></DataTag>
<Description><![CDATA[test2 test3 data2]]></Description> <Description><![CDATA[test3 test3 data2]]></Description>
</Message> </Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<DataTag><![CDATA[data2]]></DataTag> <DataTag><![CDATA[data2]]></DataTag>
@ -155,6 +155,48 @@
</Message> </Message>
<Duration msecs="0"/> <Duration msecs="0"/>
</TestFunction> </TestFunction>
<TestFunction name="multiFail">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[init multiFail (null)]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[This failure message should be repeated ten times]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
<Description><![CDATA[But this test should only contribute one to the failure count]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[cleanup multiFail (null)]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase"> <TestFunction name="cleanupTestCase">
<Message type="qdebug" file="" line="0"> <Message type="qdebug" file="" line="0">
<Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description> <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description>

18
tests/auto/testlib/selftests/subtest/tst_subtest.cpp
View File

@ -1,6 +1,6 @@
/**************************************************************************** /****************************************************************************
** **
** Copyright (C) 2016 The Qt Company Ltd. ** Copyright (C) 2021 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/ ** Contact: https://www.qt.io/licensing/
** **
** This file is part of the test suite of the Qt Toolkit. ** This file is part of the test suite of the Qt Toolkit.
@ -47,6 +47,8 @@ private slots:
void test2(); void test2();
void test3_data(); void test3_data();
void test3(); void test3();
void multiFail();
}; };
@ -131,7 +133,7 @@ void tst_Subtest::test3_data()
void tst_Subtest::test3() void tst_Subtest::test3()
{ {
qDebug() << "test2" qDebug() << "test3"
<< (QTest::currentTestFunction() ? QTest::currentTestFunction() : "(null)") << (QTest::currentTestFunction() ? QTest::currentTestFunction() : "(null)")
<< (QTest::currentDataTag() ? QTest::currentDataTag() : "(null)"); << (QTest::currentDataTag() ? QTest::currentDataTag() : "(null)");
@ -140,7 +142,17 @@ void tst_Subtest::test3()
// second and third time we call this it should FAIL // second and third time we call this it should FAIL
QCOMPARE(str, QString("hello0")); QCOMPARE(str, QString("hello0"));
qDebug() << "test2 end"; qDebug() << "test3 end";
}
void tst_Subtest::multiFail()
{
// Simulates tests which call a shared function that does common checks, or
// that do checks in code run asynchronously from a messae loop.
for (int i = 0; i < 10; ++i)
[]() { QFAIL("This failure message should be repeated ten times"); }();
// FIXME QTBUG-95661: it gets counted as eleven failures, of course.
QFAIL("But this test should only contribute one to the failure count");
} }
QTEST_MAIN(tst_Subtest) QTEST_MAIN(tst_Subtest)