diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index fd4cdb3488..31521263ee 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -1,5 +1,5 @@ - + @@ -19,66 +19,166 @@ + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + - + - - - + + + - + - - + + + + + + + + + + + - + - + + + + - + + + + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml index 7c91656b89..36636e13c4 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.lightxml +++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml @@ -30,12 +30,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -46,93 +81,181 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + - + + + + + + + + - + - + - + - + - + + + + + - + + + + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -142,7 +265,25 @@ - + + + + + + + + + + + + + + + + + + + @@ -152,6 +293,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index 2ad422b7f9..984815909a 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -16,81 +16,169 @@ not ok 3 - xfailAndAbort() # TODO This should xfail file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 4 - xfailTwice() +not ok 4 - xfailContinueSkip() # TODO This should xfail then skip + --- + at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted +not ok 5 - xfailAbortSkip() # TODO This should xfail + --- + at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 6 - xfailTwice() --- # Already expecting a fail at: tst_ExpectFail::xfailTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 5 - xfailWithQString() # TODO A string +not ok 7 - xfailDataDrivenTwice(Pass Abort) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 8 - xfailDataDrivenTwice(Pass Continue) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 9 - xfailDataDrivenTwice(Fail Abort) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 10 - xfailDataDrivenTwice(Fail Continue) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 11 - xfailWithQString() # TODO A string --- at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 5 - xfailWithQString() # TODO Bug 5 (The message) +not ok 11 - xfailWithQString() # TODO Bug 5 (The message) --- at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 6 - xfailDataDrivenWithQVerify(Pass 1) -ok 7 - xfailDataDrivenWithQVerify(Pass 2) -not ok 8 - xfailDataDrivenWithQVerify(Abort) # TODO This test should xfail +ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count +ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count +not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string + --- + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string + --- + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO Bug 5 (The message) + --- + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count +ok 16 - xfailDataDrivenWithQVerify(Pass Abort) +ok 17 - xfailDataDrivenWithQVerify(Pass Continue) +not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 9 - xfailDataDrivenWithQVerify(Continue) # TODO This test should xfail +not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 10 - xfailDataDrivenWithQCompare(Pass 1) -ok 11 - xfailDataDrivenWithQCompare(Pass 2) -not ok 12 - xfailDataDrivenWithQCompare(Abort) # TODO This test should xfail +ok 20 - xfailDataDrivenWithQCompare(Pass Abort) +ok 21 - xfailDataDrivenWithQCompare(Pass Continue) +not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 13 - xfailDataDrivenWithQCompare(Continue) # TODO This test should xfail +not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 14 - xfailOnWrongRow(right row) -not ok 15 - xfailOnAnyRow(first row) # TODO This test should xfail +# Should pass (*not* xpass), despite test-case name +ok 24 - xfailOnWrongRow(Fail Abort) +# Should pass (*not* xpass), despite test-case name +ok 25 - xfailOnWrongRow(Fail Continue) +not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail --- at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 16 - xfailOnAnyRow(second row) # TODO This test should xfail +not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail --- at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 17 - xfailWithoutVerify(first row) +# Should fail (*not* xfail), despite test-case name +not ok 28 - xfailWithoutCheck(Fail Abort) --- # QEXPECT_FAIL was called without any subsequent verification statements ... -not ok 18 - xfailWithoutVerify(second row) +# Should fail (*not* xfail), despite test-case name +not ok 29 - xfailWithoutCheck(Fail Continue) --- # QEXPECT_FAIL was called without any subsequent verification statements ... -ok 19 - xpass() # TODO 'true' returned TRUE unexpectedly. () -ok 20 - xpassDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () -ok 21 - xpassDataDrivenWithQVerify(Pass) -ok 22 - xpassDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. -ok 23 - xpassDataDrivenWithQCompare(Pass) -ok 24 - cleanupTestCase() -1..24 -# tests 24 -# pass 18 -# fail 6 +ok 30 - xpassAbort() # TODO 'true' returned TRUE unexpectedly. () +ok 31 - xpassAbortSkip() # TODO 'true' returned TRUE unexpectedly. () +ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. () +ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. () +# This should be reached +ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. () +ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count +ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () +not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals + --- + at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +ok 38 - xpassAbortDataDrivenWithQVerify(Pass) +ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +# Test should Continue past XPASS +# Test should simply PASS +ok 40 - xpassContinueDataDrivenWithQVerify(Pass) +ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +ok 42 - xpassAbortDataDrivenWithQCompare(Pass) +ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +# Test should Continue past XPASS +# Test should simply PASS +ok 44 - xpassContinueDataDrivenWithQCompare(Pass) +ok 45 - cleanupTestCase() +1..45 +# tests 45 +# pass 23 +# fail 17 diff --git a/tests/auto/testlib/selftests/expected_expectfail.teamcity b/tests/auto/testlib/selftests/expected_expectfail.teamcity index cac2c34491..738ef664ec 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.teamcity +++ b/tests/auto/testlib/selftests/expected_expectfail.teamcity @@ -7,59 +7,120 @@ ##teamcity[testStarted name='xfailAndAbort()' flowId='tst_ExpectFail'] ##teamcity[testStdOut name='xfailAndAbort()' out='QDEBUG: begin|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xfailAndAbort()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be reported and counted |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailAbortSkip()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailAbortSkip()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail then skip|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailAbortSkip()' flowId='tst_ExpectFail'] ##teamcity[testStarted name='xfailTwice()' flowId='tst_ExpectFail'] ##teamcity[testFailed name='xfailTwice()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xfailTwice()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Pass Abort)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Pass Continue)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Fail Abort)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Fail Continue)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Fail Continue)' flowId='tst_ExpectFail'] ##teamcity[testStarted name='xfailWithQString()' flowId='tst_ExpectFail'] ##teamcity[testStdOut name='xfailWithQString()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: Bug 5 (The message)' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xfailWithQString()' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailOnWrongRow(right row)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailOnWrongRow(right row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailOnAnyRow(first row)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailOnAnyRow(first row)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailOnAnyRow(first row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailOnAnyRow(second row)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailOnAnyRow(second row)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailOnAnyRow(second row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailWithoutVerify(first row)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xfailWithoutVerify(first row)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailWithoutVerify(first row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailWithoutVerify(second row)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xfailWithoutVerify(second row)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailWithoutVerify(second row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpass()' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpass()' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xpassDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xpassDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailDataDrivenWithQString(Pass Abort)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailDataDrivenWithQString(Pass Continue)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQString(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQString(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQString(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQString(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailDataDrivenWithQString(Fail Continue)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Pass Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: Bug 5 (The message)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnWrongRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnWrongRow(Fail Abort)' out='QDEBUG: Should pass (*not* xpass), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnWrongRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnWrongRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnWrongRow(Fail Continue)' out='QDEBUG: Should pass (*not* xpass), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnWrongRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnAnyRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnAnyRow(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnAnyRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnAnyRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnAnyRow(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnAnyRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailWithoutCheck(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailWithoutCheck(Fail Abort)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailWithoutCheck(Fail Abort)' out='QDEBUG: Should fail (*not* xfail), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailWithoutCheck(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailWithoutCheck(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailWithoutCheck(Fail Continue)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailWithoutCheck(Fail Continue)' out='QDEBUG: Should fail (*not* xfail), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailWithoutCheck(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbort()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbort()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortSkip()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortSkip()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortXfailContinue()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortXfailContinue()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortXfailContinue()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinue()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinue()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinue()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueSkip()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassContinueSkip()' out='QDEBUG: This should be reached' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueSkip()' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xpassContinueSkip()' message='This should be reached but not increment skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueXfailAbort()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueXfailAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueXfailAbort()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassAbortDataDrivenWithQVerify(XPass)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail but not add to totals' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassContinueDataDrivenWithQVerify(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail'] ##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail'] diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 75d41e5733..83f673f2f3 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -10,46 +10,104 @@ QDEBUG : tst_ExpectFail::xfailAndAbort() begin XFAIL : tst_ExpectFail::xfailAndAbort() This should xfail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] PASS : tst_ExpectFail::xfailAndAbort() +XFAIL : tst_ExpectFail::xfailContinueSkip() This should xfail then skip + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +SKIP : tst_ExpectFail::xfailContinueSkip() This skip should be reported and counted + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XFAIL : tst_ExpectFail::xfailAbortSkip() This should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailAbortSkip() FAIL! : tst_ExpectFail::xfailTwice() Already expecting a fail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Pass Abort) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Pass Continue) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Fail Abort) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Fail Continue) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] XFAIL : tst_ExpectFail::xfailWithQString() A string Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] XFAIL : tst_ExpectFail::xfailWithQString() Bug 5 (The message) Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] PASS : tst_ExpectFail::xfailWithQString() -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass 1) -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass 2) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Abort) This test should xfail +SKIP : tst_ExpectFail::xfailDataDrivenWithQString(Pass Abort) Each Continue or Pass reports this and increments skip-count Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Abort) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Continue) This test should xfail +SKIP : tst_ExpectFail::xfailDataDrivenWithQString(Pass Continue) Each Continue or Pass reports this and increments skip-count Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Continue) -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass 1) -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass 2) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Abort) This test should xfail +XFAIL : tst_ExpectFail::xfailDataDrivenWithQString(Fail Abort) A string Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Abort) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Continue) This test should xfail +PASS : tst_ExpectFail::xfailDataDrivenWithQString(Fail Abort) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQString(Fail Continue) A string Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Continue) -PASS : tst_ExpectFail::xfailOnWrongRow(right row) -XFAIL : tst_ExpectFail::xfailOnAnyRow(first row) This test should xfail +XFAIL : tst_ExpectFail::xfailDataDrivenWithQString(Fail Continue) Bug 5 (The message) Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailOnAnyRow(first row) -XFAIL : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail +SKIP : tst_ExpectFail::xfailDataDrivenWithQString(Fail Continue) Each Continue or Pass reports this and increments skip-count Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailOnAnyRow(second row) -FAIL! : tst_ExpectFail::xfailWithoutVerify(first row) QEXPECT_FAIL was called without any subsequent verification statements -FAIL! : tst_ExpectFail::xfailWithoutVerify(second row) QEXPECT_FAIL was called without any subsequent verification statements -XPASS : tst_ExpectFail::xpass() 'true' returned TRUE unexpectedly. () +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass Abort) +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass Continue) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Abort) This test should xfail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -XPASS : tst_ExpectFail::xpassDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. () +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Abort) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Continue) This test should xfail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xpassDataDrivenWithQVerify(Pass) -XPASS : tst_ExpectFail::xpassDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) returned TRUE unexpectedly. +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Continue) +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass Abort) +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass Continue) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Abort) This test should xfail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xpassDataDrivenWithQCompare(Pass) +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Abort) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Continue) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Continue) +QDEBUG : tst_ExpectFail::xfailOnWrongRow(Fail Abort) Should pass (*not* xpass), despite test-case name +PASS : tst_ExpectFail::xfailOnWrongRow(Fail Abort) +QDEBUG : tst_ExpectFail::xfailOnWrongRow(Fail Continue) Should pass (*not* xpass), despite test-case name +PASS : tst_ExpectFail::xfailOnWrongRow(Fail Continue) +XFAIL : tst_ExpectFail::xfailOnAnyRow(Fail Abort) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailOnAnyRow(Fail Abort) +XFAIL : tst_ExpectFail::xfailOnAnyRow(Fail Continue) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailOnAnyRow(Fail Continue) +QDEBUG : tst_ExpectFail::xfailWithoutCheck(Fail Abort) Should fail (*not* xfail), despite test-case name +FAIL! : tst_ExpectFail::xfailWithoutCheck(Fail Abort) QEXPECT_FAIL was called without any subsequent verification statements +QDEBUG : tst_ExpectFail::xfailWithoutCheck(Fail Continue) Should fail (*not* xfail), despite test-case name +FAIL! : tst_ExpectFail::xfailWithoutCheck(Fail Continue) QEXPECT_FAIL was called without any subsequent verification statements +XPASS : tst_ExpectFail::xpassAbort() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassAbortSkip() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassAbortXfailContinue() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassContinue() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +QDEBUG : tst_ExpectFail::xpassContinue() This should be reached +XPASS : tst_ExpectFail::xpassContinueSkip() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +SKIP : tst_ExpectFail::xpassContinueSkip() This should be reached but not increment skip-count + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassContinueXfailAbort() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XFAIL : tst_ExpectFail::xpassContinueXfailAbort() This test should xfail but not add to totals + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassAbortDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xpassAbortDataDrivenWithQVerify(Pass) +XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(XPass) Test should Continue past XPASS +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(Pass) Test should simply PASS +PASS : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(Pass) +XPASS : tst_ExpectFail::xpassAbortDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) returned TRUE unexpectedly. + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xpassAbortDataDrivenWithQCompare(Pass) +XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) returned TRUE unexpectedly. + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) Test should Continue past XPASS +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should simply PASS +PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) PASS : tst_ExpectFail::cleanupTestCase() -Totals: 18 passed, 6 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml index ccfd942c92..27aae6b75c 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.xml +++ b/tests/auto/testlib/selftests/expected_expectfail.xml @@ -32,12 +32,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48,93 +83,181 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + - + + + + + + + + - + - + - + - + - + + + + + - + + + + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -144,7 +267,25 @@ - + + + + + + + + + + + + + + + + + + + @@ -154,6 +295,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index fb6300b4a8..b97244b476 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the test suite of the Qt Toolkit. @@ -39,23 +39,42 @@ class tst_ExpectFail: public QObject private slots: void xfailAndContinue() const; void xfailAndAbort() const; + void xfailContinueSkip() const; + void xfailAbortSkip() const; void xfailTwice() const; + void xfailDataDrivenTwice() const; + void xfailDataDrivenTwice_data() const { xfailDataDriven_data(false); } void xfailWithQString() const; - void xfailDataDrivenWithQVerify_data() const; + void xfailDataDrivenWithQString_data() const { xfailDataDriven_data(false); } + void xfailDataDrivenWithQString() const; + void xfailDataDrivenWithQVerify_data() const { xfailDataDriven_data(false); } void xfailDataDrivenWithQVerify() const; - void xfailDataDrivenWithQCompare_data() const; + void xfailDataDrivenWithQCompare_data() const { xfailDataDriven_data(false); } void xfailDataDrivenWithQCompare() const; - void xfailOnWrongRow_data() const; + void xfailOnWrongRow_data() const { xfailDataDriven_data(true); } void xfailOnWrongRow() const; - void xfailOnAnyRow_data() const; + void xfailOnAnyRow_data() const { xfailDataDriven_data(true); } void xfailOnAnyRow() const; - void xfailWithoutVerify_data() const; - void xfailWithoutVerify() const; - void xpass() const; - void xpassDataDrivenWithQVerify_data() const; - void xpassDataDrivenWithQVerify() const; - void xpassDataDrivenWithQCompare_data() const; - void xpassDataDrivenWithQCompare() const; + void xfailWithoutCheck_data() const { xfailDataDriven_data(true); } + void xfailWithoutCheck() const; + void xpassAbort() const; + void xpassAbortSkip() const; + void xpassAbortXfailContinue() const; + void xpassContinue() const; + void xpassContinueSkip() const; + void xpassContinueXfailAbort() const; + void xpassAbortDataDrivenWithQVerify_data() const { xpassDataDriven_data(); } + void xpassAbortDataDrivenWithQVerify() const; + void xpassContinueDataDrivenWithQVerify_data() const { xpassDataDriven_data(); } + void xpassContinueDataDrivenWithQVerify() const; + void xpassAbortDataDrivenWithQCompare_data() const { xpassDataDriven_data(); } + void xpassAbortDataDrivenWithQCompare() const; + void xpassContinueDataDrivenWithQCompare_data() const { xpassDataDriven_data(); } + void xpassContinueDataDrivenWithQCompare() const; + +private: + void xfailDataDriven_data(bool failOnly) const; + void xpassDataDriven_data() const; }; void tst_ExpectFail::xfailAndContinue() const @@ -76,11 +95,48 @@ void tst_ExpectFail::xfailAndAbort() const QVERIFY2(false, "This should not be reached"); } +void tst_ExpectFail::xfailContinueSkip() const +{ + QEXPECT_FAIL("", "This should xfail then skip", Continue); + QVERIFY(false); + QSKIP("This skip should be reported and counted"); +} + +void tst_ExpectFail::xfailAbortSkip() const +{ + QEXPECT_FAIL("", "This should xfail", Abort); + QVERIFY(false); + + // If we get here the test did not correctly abort on the previous QVERIFY. + QSKIP("This skip should not be reached"); +} + void tst_ExpectFail::xfailTwice() const { - QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Abort); + QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue); QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is " - "an error and should abort this test function", Abort); + "an error and should abort this test function", Continue); + + // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL. + QVERIFY2(false, "This should not be reached"); +} + +void tst_ExpectFail::xfailDataDrivenTwice() const +{ + // Same with data-driven cases (twist semantics of unused shouldPass; we + // have four combinations to test): + QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); + QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Continue); + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Abort); + QEXPECT_FAIL("Pass Continue", "Calling QEXPECT_FAIL once on a test-case is fine", Continue); + QEXPECT_FAIL("Pass Continue", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Abort); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL once on a test-case is fine", Continue); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Continue); // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL. QVERIFY2(false, "This should not be reached"); @@ -92,124 +148,119 @@ void tst_ExpectFail::xfailWithQString() const QVERIFY(false); int bugNo = 5; - QString msg("The message"); - QEXPECT_FAIL( "", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue); + QLatin1String msg("The message"); + QEXPECT_FAIL("", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue); QVERIFY(false); } -void tst_ExpectFail::xfailDataDrivenWithQVerify_data() const +void tst_ExpectFail::xfailDataDrivenWithQString() const { - QTest::addColumn("shouldPass"); - QTest::addColumn("failMode"); + // This test does not (yet) distinguish the two Pass cases. + QFETCH(bool, shouldPass); - QTest::newRow("Pass 1") << true << QTest::Abort; - QTest::newRow("Pass 2") << true << QTest::Continue; - QTest::newRow("Abort") << false << QTest::Abort; - QTest::newRow("Continue") << false << QTest::Continue; + QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort); + QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue); + // TODO: why aren't QVERIFY2()'s smessages seen ? + QVERIFY2(shouldPass, "Both Fail cases should XFAIL here"); + // Fail Abort should be gone now. + + int bugNo = 5; + QLatin1String msg("The message"); + QEXPECT_FAIL("Fail Continue", qPrintable(QString("Bug %1 (%2)").arg(bugNo).arg(msg)), Continue); + QVERIFY2(shouldPass, "Only Fail Continue should see this"); + + // FAIL is a pass, and SKIP trumps pass. + QSKIP("Each Continue or Pass reports this and increments skip-count"); } void tst_ExpectFail::xfailDataDrivenWithQVerify() const { + // This test does not (yet) distinguish the two Pass cases. QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); - // You can't pass a variable as the last parameter of QEXPECT_FAIL, - // because the macro adds "QTest::" in front of the last parameter. - // That is why the following code appears to be a little strange. - if (!shouldPass) { - if (failMode == QTest::Abort) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort); - else - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue); - } + QEXPECT_FAIL("Fail Abort", "This test should xfail", Abort); + QEXPECT_FAIL("Fail Continue", "This test should xfail", Continue); + QVERIFY2(shouldPass, "Both Fail cases should XFAIL here"); - QVERIFY(shouldPass); - - // If we get here, we either expected to pass or we expected to + // If we get here, either we expected to pass or we expected to // fail and the failure mode was Continue. if (!shouldPass) QCOMPARE(failMode, QTest::Continue); } -void tst_ExpectFail::xfailDataDrivenWithQCompare_data() const +void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const { QTest::addColumn("shouldPass"); QTest::addColumn("failMode"); - QTest::newRow("Pass 1") << true << QTest::Abort; - QTest::newRow("Pass 2") << true << QTest::Continue; - QTest::newRow("Abort") << false << QTest::Abort; - QTest::newRow("Continue") << false << QTest::Continue; + if (!failOnly) { + QTest::newRow("Pass Abort") << true << QTest::Abort; + QTest::newRow("Pass Continue") << true << QTest::Continue; + } + QTest::newRow("Fail Abort") << false << QTest::Abort; + QTest::newRow("Fail Continue") << false << QTest::Continue; } void tst_ExpectFail::xfailDataDrivenWithQCompare() const { + // This test does not (yet) distinguish the two Pass cases. QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); - // You can't pass a variable as the last parameter of QEXPECT_FAIL, - // because the macro adds "QTest::" in front of the last parameter. - // That is why the following code appears to be a little strange. - if (!shouldPass) { - if (failMode == QTest::Abort) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort); - else - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue); - } + QEXPECT_FAIL("Fail Abort", "This test should xfail", Abort); + QEXPECT_FAIL("Fail Continue", "This test should xfail", Continue); QCOMPARE(1, shouldPass ? 1 : 2); - // If we get here, we either expected to pass or we expected to + // If we get here, either we expected to pass or we expected to // fail and the failure mode was Continue. if (!shouldPass) QCOMPARE(failMode, QTest::Continue); } -void tst_ExpectFail::xfailOnWrongRow_data() const -{ - QTest::addColumn("dummy"); - - QTest::newRow("right row") << 0; -} - void tst_ExpectFail::xfailOnWrongRow() const { - // QEXPECT_FAIL for a row that is not the current row should be ignored. - QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort); - QVERIFY(true); -} - -void tst_ExpectFail::xfailOnAnyRow_data() const -{ - QTest::addColumn("dummy"); - - QTest::newRow("first row") << 0; - QTest::newRow("second row") << 1; + qDebug("Should pass (*not* xpass), despite test-case name"); + // QEXPECT_FAIL for a row that does not exist should be ignored. + // (It might be conditional data(), so exist in other circumstances.) + QFETCH(QTest::TestFailMode, failMode); + // You can't pass a variable as the last parameter of QEXPECT_FAIL, + // because the macro adds "QTest::" in front of the last parameter. + // That is why the following code appears to be a little strange. + if (failMode == QTest::Abort) + QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort); + else + QEXPECT_FAIL("wrong row", "This xfail should be ignored", Continue); + QTEST(false, "shouldPass"); // _data skips the passing tests as pointless } void tst_ExpectFail::xfailOnAnyRow() const { // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL // should mean that the failure is expected for all data rows. - QEXPECT_FAIL("", "This test should xfail", Abort); - QVERIFY(false); + QFETCH(QTest::TestFailMode, failMode); + // You can't pass a variable as the last parameter of QEXPECT_FAIL, + // because the macro adds "QTest::" in front of the last parameter. + // That is why the following code appears to be a little strange. + if (failMode == QTest::Abort) + QEXPECT_FAIL("", "This test should xfail", Abort); + else + QEXPECT_FAIL("", "This test should xfail", Continue); + QTEST(true, "shouldPass"); // _data skips the passing tests as pointless } -void tst_ExpectFail::xfailWithoutVerify_data() const +void tst_ExpectFail::xfailWithoutCheck() const { - QTest::addColumn("dummy"); - - QTest::newRow("first row") << 0; - QTest::newRow("second row") << 1; + qDebug("Should fail (*not* xfail), despite test-case name"); + QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error", + Abort); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL without any subsequent check is an error", + Continue); } -void tst_ExpectFail::xfailWithoutVerify() const -{ - QVERIFY(true); - QEXPECT_FAIL("", "This expected failure should be ignored", Abort); -} - -void tst_ExpectFail::xpass() const +void tst_ExpectFail::xpassAbort() const { QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -219,7 +270,33 @@ void tst_ExpectFail::xpass() const QVERIFY2(false, "This should not be reached"); } -void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const +void tst_ExpectFail::xpassAbortSkip() const +{ + QEXPECT_FAIL("", "This test should xpass", Abort); + QVERIFY(true); + + QSKIP("This should not be reached (and not add to skip-count)"); +} + +void tst_ExpectFail::xpassAbortXfailContinue() const +{ + QEXPECT_FAIL("", "This test should xpass", Abort); + QVERIFY(true); + + // If we get here the test did not correctly abort on the previous + // unexpected pass. + QEXPECT_FAIL("", "This should not be reached", Continue); + QVERIFY2(false, "This should not be reached"); +} + +void tst_ExpectFail::xpassContinue() const +{ + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + qDebug("This should be reached"); +} + +void tst_ExpectFail::xpassDataDriven_data() const { QTest::addColumn("shouldXPass"); @@ -227,38 +304,62 @@ void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const QTest::newRow("Pass") << false; } -void tst_ExpectFail::xpassDataDrivenWithQVerify() const +void tst_ExpectFail::xpassContinueSkip() const +{ + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + QSKIP("This should be reached but not increment skip-count"); +} + +void tst_ExpectFail::xpassContinueXfailAbort() const +{ + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort); + QVERIFY(false); + QVERIFY2(false, "This should not be reached"); +} + +void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const { QFETCH(bool, shouldXPass); - if (shouldXPass) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort); - + QEXPECT_FAIL("XPass", "This test-row should xpass", Abort); QVERIFY(true); // We should only get here if the test wasn't supposed to xpass. - QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS"); + QVERIFY2(!shouldXPass, "Test failed to Abort on XPASS"); } -void tst_ExpectFail::xpassDataDrivenWithQCompare_data() const -{ - QTest::addColumn("shouldXPass"); - - QTest::newRow("XPass") << true; - QTest::newRow("Pass") << false; -} - -void tst_ExpectFail::xpassDataDrivenWithQCompare() const +void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const { QFETCH(bool, shouldXPass); - if (shouldXPass) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort); + QEXPECT_FAIL("XPass", "This test-row should xpass", Continue); + QVERIFY(true); + qDebug(shouldXPass ? "Test should Continue past XPASS" : "Test should simply PASS"); +} + +void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const +{ + QFETCH(bool, shouldXPass); + + QEXPECT_FAIL("XPass", "This test should xpass", Abort); QCOMPARE(1, 1); // We should only get here if the test wasn't supposed to xpass. - QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS"); + QVERIFY2(!shouldXPass, "Test failed to Abort on XPASS"); +} + +void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const +{ + QFETCH(bool, shouldXPass); + + QEXPECT_FAIL("XPass", "This test should xpass", Continue); + QCOMPARE(1, 1); + + qDebug(shouldXPass ? "Test should Continue past XPASS" : "Test should simply PASS"); } QTEST_MAIN(tst_ExpectFail)