AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
95e4996688
qt5base-lts/tests/auto/testlib/selftests/expected_expectfail.teamcity
Edward Welbourne 77a93e6df3 Make Skip an incident in test logging 
Skip ends the test (albeit inconclusively).  Rearrange the enums in
the abstract logger, move code to handle skip between relevant
function and tidy up various things that became simpler as a result.

Also reorder the message enum, and its switches, to separate testlib's
internals from the usual Qt messages, and put each group in ascending
order of severity.

Task-number: QTBUG-96844
Change-Id: I2c7a634b9f849830d64eafa750155e66e244b729
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
2021-12-09 20:54:26 +01:00

136 lines
17 KiB
Plaintext
Raw Blame History

##teamcity[testSuiteStarted name='tst_ExpectFail' flowId='tst_ExpectFail']
##teamcity[testStarted name='initTestCase()' flowId='tst_ExpectFail']
##teamcity[testFinished name='initTestCase()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailAndContinue()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailAndContinue()' out='QDEBUG: begin|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail|nQDEBUG: after' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailAndContinue()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailAndAbort()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailAndAbort()' out='QDEBUG: begin|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailAndAbort()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_ExpectFail']
##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be reported and counted |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailContinueSkip()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail then skip' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailContinueSkip()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailAbortSkip()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailAbortSkip()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailAbortSkip()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailTwice()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailTwice()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailTwice()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenTwice(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailDataDrivenTwice(Pass Abort)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenTwice(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenTwice(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailDataDrivenTwice(Pass Continue)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenTwice(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenTwice(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailDataDrivenTwice(Fail Abort)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenTwice(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenTwice(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailDataDrivenTwice(Fail Continue)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenTwice(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailWithQString()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailWithQString()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: Bug 5 (The message)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailWithQString()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQString(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testIgnored name='xfailDataDrivenWithQString(Pass Abort)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQString(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQString(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testIgnored name='xfailDataDrivenWithQString(Pass Continue)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQString(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQString(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailDataDrivenWithQString(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQString(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQString(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testIgnored name='xfailDataDrivenWithQString(Fail Continue)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailDataDrivenWithQString(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: Bug 5 (The message)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQString(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQVerify(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQVerify(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQVerify(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQVerify(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQCompare(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQCompare(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailDataDrivenWithQCompare(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailDataDrivenWithQCompare(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailOnWrongRow(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailOnWrongRow(Fail Abort)' out='QDEBUG: Should pass (*not* xpass), despite test-case name' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailOnWrongRow(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailOnWrongRow(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailOnWrongRow(Fail Continue)' out='QDEBUG: Should pass (*not* xpass), despite test-case name' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailOnWrongRow(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailOnAnyRow(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailOnAnyRow(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailOnAnyRow(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailOnAnyRow(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailOnAnyRow(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailOnAnyRow(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailWithoutCheck(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailWithoutCheck(Fail Abort)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailWithoutCheck(Fail Abort)' out='QDEBUG: Should fail (*not* xfail), despite test-case name' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailWithoutCheck(Fail Abort)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xfailWithoutCheck(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailWithoutCheck(Fail Continue)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xfailWithoutCheck(Fail Continue)' out='QDEBUG: Should fail (*not* xfail), despite test-case name' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailWithoutCheck(Fail Continue)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbort()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbort()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbortSkip()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassAbortSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbortSkip()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbortXfailContinue()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassAbortXfailContinue()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbortXfailContinue()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinue()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassContinue()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinue()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinueSkip()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassContinueSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xpassContinueSkip()' out='QDEBUG: This should be reached' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueSkip()' flowId='tst_ExpectFail']
##teamcity[testIgnored name='xpassContinueSkip()' message='This should be reached but not increment skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueSkip()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinueXfailAbort()' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassContinueXfailAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueXfailAbort()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbortDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassAbortDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xpassAbortDataDrivenWithQVerify(XPass)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail but not add to totals' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbortDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbortDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbortDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinueDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassContinueDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinueDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xpassContinueDataDrivenWithQVerify(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbortDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassAbortDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbortDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassAbortDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassAbortDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinueDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassContinueDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail']
##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail']
##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 44: 23 passed, 17 failed, 4 skipped' flowId='tst_ExpectFail']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail']
##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail']
Reference in New Issue View Git Blame Copy Permalink