AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
93830d9ed3
qt5base-lts/tests/auto/testlib/selftests/expected_expectfail.tap
Edward Welbourne be72cb9cfd Include check for the Totals line in testlib's expectedfail selftest 
Count how many we expect to pass, fail and be skipped, so we can
report that for comparison with the actual totals line: this reveals
some double-counting, marked with a FIXME.

Task-number: QTBUG-95661
Change-Id: I8b0f13ded5202ed476d8abfee70ed60f9e639bf9
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
2021-08-27 20:16:18 +02:00

186 lines
8.3 KiB
Plaintext
Raw Blame History

TAP version 13
# tst_ExpectFail
ok 1 - initTestCase()
# begin
not ok 2 - xfailAndContinue() # TODO This should xfail
---
at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
# after
# begin
not ok 3 - xfailAndAbort() # TODO This should xfail
---
at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 4 - xfailContinueSkip() # TODO This should xfail then skip
---
at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted
not ok 5 - xfailAbortSkip() # TODO This should xfail
---
at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 6 - xfailTwice()
---
# Already expecting a fail
at: tst_ExpectFail::xfailTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 7 - xfailDataDrivenTwice(Pass Abort)
---
# Already expecting a fail
at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 8 - xfailDataDrivenTwice(Pass Continue)
---
# Already expecting a fail
at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 9 - xfailDataDrivenTwice(Fail Abort)
---
# Already expecting a fail
at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 10 - xfailDataDrivenTwice(Fail Continue)
---
# Already expecting a fail
at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 11 - xfailWithQString() # TODO A string
---
at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 11 - xfailWithQString() # TODO Bug 5 (The message)
---
at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count
ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count
not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string
---
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string
---
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO Bug 5 (The message)
---
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count
ok 16 - xfailDataDrivenWithQVerify(Pass Abort)
ok 17 - xfailDataDrivenWithQVerify(Pass Continue)
not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail
---
at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail
---
at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
ok 20 - xfailDataDrivenWithQCompare(Pass Abort)
ok 21 - xfailDataDrivenWithQCompare(Pass Continue)
not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail
---
at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail
---
at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
# Should pass (*not* xpass), despite test-case name
ok 24 - xfailOnWrongRow(Fail Abort)
# Should pass (*not* xpass), despite test-case name
ok 25 - xfailOnWrongRow(Fail Continue)
not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail
---
at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail
---
at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
# Should fail (*not* xfail), despite test-case name
not ok 28 - xfailWithoutCheck(Fail Abort)
---
# QEXPECT_FAIL was called without any subsequent verification statements
...
# Should fail (*not* xfail), despite test-case name
not ok 29 - xfailWithoutCheck(Fail Continue)
---
# QEXPECT_FAIL was called without any subsequent verification statements
...
ok 30 - xpassAbort() # TODO 'true' returned TRUE unexpectedly. ()
ok 31 - xpassAbortSkip() # TODO 'true' returned TRUE unexpectedly. ()
ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. ()
ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. ()
# This should be reached
ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. ()
ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count
ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. ()
not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals
---
at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
ok 38 - xpassAbortDataDrivenWithQVerify(Pass)
ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
# Test should Continue past XPASS
# Test should simply PASS
ok 40 - xpassContinueDataDrivenWithQVerify(Pass)
ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
ok 42 - xpassAbortDataDrivenWithQCompare(Pass)
ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
# Test should Continue past XPASS
# Test should simply PASS
ok 44 - xpassContinueDataDrivenWithQCompare(Pass)
# Totals should add up to 44: 23 passed, 17 failed, 4 skipped
ok 45 - cleanupTestCase()
1..45
# tests 45
# pass 23
# fail 17
Reference in New Issue View Git Blame Copy Permalink