Add more tests of blacklisting, combined with XPASS and XFAIL

Include counting of test types, to catch two more cases where totals
don't add up.

Task-number: QTBUG-95661
Change-Id: I9fe5424bc6652c61a065bf2889333e2ed9437c81
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

tests/auto/testlib/selftests/blacklisted/BLACKLIST

@@ -12,5 +12,17 @@ obscure # no such platform; is ignored
 [xfail]
 *
 
+[xfailContinueSkip]
+*
+
+[xfailContinueFail]
+*
+
 [xpass]
 *
+
+[xpassContinueSkip]
+*
+
+[xpassContinueFail]
+*

tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp

@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2021 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the test suite of the Qt Toolkit.
@@ -35,41 +35,104 @@ class tst_Blacklisted : public QObject
     Q_OBJECT
 
 private slots:
+    void init();
+    void cleanupTestCase();
     void pass();
     void skip();
     void fail();
     void xfail();
+    void xfailContinueSkip();
+    void xfailContinueFail();
     void xpass();
+    void xpassContinueSkip();
+    void xpassContinueFail();
+
+private:
+    int casesTested = 2;
+    // What the totals line's numbers *should* be:
+    int passed = 2, skipped = 0, blacklisted = 0;
+    // Total and passed get {init,cleanup}TestCase() in addition to the actual tests.
 };
 
+void tst_Blacklisted::init()
+{
+    ++casesTested;
+}
+
+void tst_Blacklisted::cleanupTestCase()
+{
+    qDebug("Totals should add up to %d: %d passed, 0 failed, %d skipped, %d blacklisted",
+           casesTested, passed, skipped, blacklisted);
+}
+
 // All the tests below have been blacklisted in blacklisted/BLACKLIST
 
 void tst_Blacklisted::pass()
 {
+    ++blacklisted;
+    qDebug("This test should BPASS");
     QVERIFY(true);
 }
 
 void tst_Blacklisted::skip()
 {
+    ++skipped;
     QSKIP("This test should SKIP");
 }
 
 void tst_Blacklisted::fail()
 {
+    ++blacklisted;
     QVERIFY2(false, "This test should BFAIL");
 }
 
 void tst_Blacklisted::xfail()
 {
+    ++blacklisted;
     QEXPECT_FAIL("", "This test should BXFAIL then BPASS", Abort);
     QVERIFY(false);
 }
 
+void tst_Blacklisted::xfailContinueSkip()
+{
+    ++skipped;
+    QEXPECT_FAIL("", "This test should BXFAIL then SKIP", Continue);
+    QVERIFY(false);
+    QSKIP("This skip should be seen and counted");
+}
+
+void tst_Blacklisted::xfailContinueFail()
+{
+    ++blacklisted;
+    QEXPECT_FAIL("", "This test should BXFAIL then BFAIL", Continue);
+    QVERIFY(false);
+    QFAIL("This fail should be seen and counted as blacklisted");
+}
+
 void tst_Blacklisted::xpass()
 {
+    ++blacklisted;
     QEXPECT_FAIL("", "This test should BXPASS", Abort);
     QVERIFY2(true, "This test should BXPASS");
 }
 
+void tst_Blacklisted::xpassContinueSkip()
+{
+    ++blacklisted;
+    QEXPECT_FAIL("", "This test should BXPASS then SKIP", Continue);
+    QVERIFY2(true, "This test should BXPASS then SKIP");
+    // FIXME QTBUG-95661: skip gets counted
+    QSKIP("This skip should be seen but not counted");
+}
+
+void tst_Blacklisted::xpassContinueFail()
+{
+    ++blacklisted;
+    QEXPECT_FAIL("", "This test should BXPASS then BFAIL", Continue);
+    QVERIFY2(true, "This test should BXPASS then BFAIL");
+    // FIXME QTBUG-95661: gets double-counted
+    QFAIL("This fail should be seen and not counted (due to prior XPASS)");
+}
+
 QTEST_MAIN(tst_Blacklisted)
 #include "tst_blacklisted.moc"

tests/auto/testlib/selftests/expected_blacklisted.junitxml

@@ -1,17 +1,33 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="0" errors="0" skipped="1" time="@TEST_DURATION@">
+<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="0" errors="0" skipped="3" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtBuild" value=""/>
   </properties>
   <testcase name="initTestCase" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
-  <testcase name="pass" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
+  <testcase name="pass" classname="tst_Blacklisted" time="@TEST_DURATION@">
+    <system-out>
+      <![CDATA[This test should BPASS]]>
+    </system-out>
+  </testcase>
   <testcase name="skip" classname="tst_Blacklisted" time="@TEST_DURATION@">
     <skipped message="This test should SKIP"/>
   </testcase>
   <testcase name="fail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
   <testcase name="xfail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
+  <testcase name="xfailContinueSkip" classname="tst_Blacklisted" time="@TEST_DURATION@">
+    <skipped message="This skip should be seen and counted"/>
+  </testcase>
+  <testcase name="xfailContinueFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
   <testcase name="xpass" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
-  <testcase name="cleanupTestCase" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
+  <testcase name="xpassContinueSkip" classname="tst_Blacklisted" time="@TEST_DURATION@">
+    <skipped message="This skip should be seen but not counted"/>
+  </testcase>
+  <testcase name="xpassContinueFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/>
+  <testcase name="cleanupTestCase" classname="tst_Blacklisted" time="@TEST_DURATION@">
+    <system-out>
+      <![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]>
+    </system-out>
+  </testcase>
 </testsuite>

tests/auto/testlib/selftests/expected_blacklisted.lightxml

@@ -8,6 +8,9 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="pass">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[This test should BPASS]]></Description>
+</Message>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
@@ -30,13 +33,52 @@
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="xfailContinueSkip">
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This test should BXFAIL then SKIP]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This skip should be seen and counted]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xfailContinueFail">
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This test should BXFAIL then BFAIL]]></Description>
+</Incident>
+<Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This fail should be seen and counted as blacklisted]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="xpass">
 <Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
     <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="xpassContinueSkip">
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This skip should be seen but not counted]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassContinueFail">
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL)]]></Description>
+</Incident>
+<Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This fail should be seen and not counted (due to prior XPASS)]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description>
+</Message>
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>

tests/auto/testlib/selftests/expected_blacklisted.tap

@@ -1,6 +1,7 @@
 TAP version 13
 # tst_Blacklisted
 ok 1 - initTestCase()
+# This test should BPASS
 ok 2 - pass() # TODO 
 ok 3 - skip() # SKIP This test should SKIP
 not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL)
@@ -22,9 +23,42 @@ not ok 5 - xfail() # TODO This test should BXFAIL then BPASS
   file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
   line: 0
   ...
-ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS)
+not ok 6 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP
-ok 7 - cleanupTestCase()
+  ---
-1..7
+  # This test should BXFAIL then SKIP
-# tests 7
+  at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+ok 6 - xfailContinueSkip() # SKIP This skip should be seen and counted
+not ok 7 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL
+  ---
+  # This test should BXFAIL then BFAIL
+  at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+not ok 7 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted
+  ---
+  # This fail should be seen and counted as blacklisted
+  at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+ok 8 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS)
+ok 9 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)
+ok 10 - xpassContinueSkip() # SKIP This skip should be seen but not counted
+ok 11 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL)
+not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted (due to prior XPASS)
+  ---
+  # This fail should be seen and not counted (due to prior XPASS)
+  at: tst_Blacklisted::xpassContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 0
+  ...
+# Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted
+ok 13 - cleanupTestCase()
+1..13
+# tests 13
 # pass 2
 # fail 0

tests/auto/testlib/selftests/expected_blacklisted.teamcity

@@ -2,6 +2,7 @@
 ##teamcity[testStarted name='initTestCase()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='initTestCase()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='pass()' flowId='tst_Blacklisted']
+##teamcity[testStdOut name='pass()' out='QDEBUG: This test should BPASS' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='pass()' flowId='tst_Blacklisted']
 ##teamcity[testIgnored name='skip()' message='This test should SKIP |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted']
@@ -9,8 +10,21 @@
 ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
+##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xfailContinueSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be seen and counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testStarted name='xfailContinueFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xfailContinueFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xfailContinueFail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted']
+##teamcity[testStarted name='xpassContinueSkip()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xpassContinueSkip()' flowId='tst_Blacklisted']
+##teamcity[testIgnored name='xpassContinueSkip()' message='This skip should be seen but not counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted']
+##teamcity[testStarted name='xpassContinueFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Blacklisted']
+##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Blacklisted']
 ##teamcity[testSuiteFinished name='tst_Blacklisted' flowId='tst_Blacklisted']

tests/auto/testlib/selftests/expected_blacklisted.txt

@@ -1,6 +1,7 @@
 ********* Start testing of tst_Blacklisted *********
 Config: Using QtTest library
 PASS   : tst_Blacklisted::initTestCase()
+QDEBUG : tst_Blacklisted::pass() This test should BPASS
 BPASS  : tst_Blacklisted::pass()
 SKIP   : tst_Blacklisted::skip() This test should SKIP
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
@@ -9,8 +10,25 @@ BFAIL  : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL
 BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BPASS  : tst_Blacklisted::xfail()
+BXFAIL : tst_Blacklisted::xfailContinueSkip() This test should BXFAIL then SKIP
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::xfailContinueSkip() This skip should be seen and counted
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BXFAIL : tst_Blacklisted::xfailContinueFail() This test should BXFAIL then BFAIL
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::xfailContinueFail() This fail should be seen and counted as blacklisted
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BXPASS : tst_Blacklisted::xpassContinueSkip() 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+SKIP   : tst_Blacklisted::xpassContinueSkip() This skip should be seen but not counted
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BXPASS : tst_Blacklisted::xpassContinueFail() 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL)
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+BFAIL  : tst_Blacklisted::xpassContinueFail() This fail should be seen and not counted (due to prior XPASS)
+   Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
+QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted
 PASS   : tst_Blacklisted::cleanupTestCase()
-Totals: 2 passed, 0 failed, 1 skipped, 4 blacklisted, 0ms
+Totals: 2 passed, 0 failed, 3 skipped, 8 blacklisted, 0ms
 ********* Finished testing of tst_Blacklisted *********

tests/auto/testlib/selftests/expected_blacklisted.xml

@@ -10,6 +10,9 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="pass">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[This test should BPASS]]></Description>
+</Message>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
@@ -32,13 +35,52 @@
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="xfailContinueSkip">
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This test should BXFAIL then SKIP]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This skip should be seen and counted]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xfailContinueFail">
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This test should BXFAIL then BFAIL]]></Description>
+</Incident>
+<Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This fail should be seen and counted as blacklisted]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="xpass">
 <Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
     <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="xpassContinueSkip">
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This skip should be seen but not counted]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassContinueFail">
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL)]]></Description>
+</Incident>
+<Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This fail should be seen and not counted (due to prior XPASS)]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description>
+</Message>
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>