From a71b9c937719bc3a5f4de6e91dbdbe364df38cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Sun, 1 Aug 2021 15:17:17 +0200 Subject: [PATCH] testlib: Clean up tst_blacklist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 33d7f76f0e847d7e0fb00dd6056e7bba45b8b1e7 attempted to exercise more of the blacklisting code, but in doing so didn't account for the fact that the features it added didn't actually work. Adding a global blacklisting of * will result in blacklisting all tests, regardless of any inverse conditions "!*" later on, such as for the 'pass' test. This would have been evident by looking at the expectation files, which still showed the test as a BPASS. The broken feature should be fixed, but in the meantime let's clean up the test so it doesn't misrepresent reality. At the same time we also remove the messages() tests, which was copied straight out of tst_silent, but doesn't serve any purpose for the blacklist selftest. Pick-to: 6.2 Change-Id: I20e4693300aad5e9ff5f17ad22e3dfcb5c49fd0e Reviewed-by: Mårten Nordheim --- .../testlib/selftests/blacklisted/BLACKLIST | 6 +--- .../selftests/blacklisted/tst_blacklisted.cpp | 36 +------------------ .../selftests/expected_blacklisted.junitxml | 25 +++---------- .../selftests/expected_blacklisted.lightxml | 27 ++------------ .../selftests/expected_blacklisted.tap | 17 ++------- .../selftests/expected_blacklisted.teamcity | 5 ++- .../selftests/expected_blacklisted.txt | 14 ++------ .../selftests/expected_blacklisted.xml | 27 ++------------ 8 files changed, 16 insertions(+), 141 deletions(-) diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index a923c11416..d652129db3 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -1,8 +1,7 @@ obscure # no such platform; is ignored -* [pass] -!* +* [skip] * @@ -15,6 +14,3 @@ obscure # no such platform; is ignored [xpass] * - -[messages] -* diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index 86f445b926..0ac6ae19f4 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -40,13 +40,9 @@ private slots: void fail(); void xfail(); void xpass(); - - // This test function must be last, as it calls qFatal(). - void messages(); }; -// All the tests below except pass() have been blacklisted in blacklisted/BLACKLIST -// Contrast with ../silent/, for the same tests without blacklisting but with -silent +// All the tests below have been blacklisted in blacklisted/BLACKLIST void tst_Blacklisted::pass() { @@ -75,35 +71,5 @@ void tst_Blacklisted::xpass() QVERIFY2(true, "This test should BXPASS"); } -#ifndef Q_OS_WIN -#include -#include - -static jmp_buf state; -static void abort_handler(int) -{ - longjmp(state, 1); -} -#endif - -void tst_Blacklisted::messages() -{ - qWarning("This is a warning that should not appear in silent test output"); - QTestLog::warn("This is an internal testlib warning that should not appear in silent test output", __FILE__, __LINE__); - qDebug("This is a debug message that should not appear in silent test output"); - qCritical("This is a critical message that should not appear in silent test output"); - qInfo("This is an info message that should not appear in silent test output"); - QTestLog::info("This is an internal testlib info message that should not appear in silent test output", __FILE__, __LINE__); - -#ifndef Q_OS_WIN - // We're testing qFatal, but we don't want to actually std::abort() ! - auto prior = signal(SIGABRT, abort_handler); - if (setjmp(state)) - signal(SIGABRT, prior); - else -#endif - qFatal("This is a fatal error message that should still appear in silent test output"); -} - QTEST_MAIN(tst_Blacklisted) #include "tst_blacklisted.moc" diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index cf32c0fa55..0051659da9 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -1,5 +1,5 @@ - + @@ -13,24 +13,7 @@ - - - - - - - - - - - - - - - - - - - - + + + diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index a692badea1..c05977661e 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -36,31 +36,8 @@ - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap index 9d589a6552..fee7750a12 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.tap +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -23,21 +23,8 @@ not ok 6 - xfail() # TODO This test should BXFAIL then BPASS line: 0 ... ok 7 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) -# This is a warning that should not appear in silent test output -# This is an internal testlib warning that should not appear in silent test output -# This is a debug message that should not appear in silent test output -# This is a critical message that should not appear in silent test output -# This is an info message that should not appear in silent test output -# This is an internal testlib info message that should not appear in silent test output -# This is a fatal error message that should still appear in silent test output -not ok 8 - messages() # TODO Received a fatal error. - --- - # Received a fatal error. - at: tst_Blacklisted::messages() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) - file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp - line: 0 - ... +ok 8 - cleanupTestCase() 1..8 # tests 8 -# pass 1 +# pass 2 # fail 0 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index 42b75b752b..ffdd02c9c2 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -11,7 +11,6 @@ ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted'] -##teamcity[testStarted name='messages()' flowId='tst_Blacklisted'] -##teamcity[testStdOut name='messages()' out='QWARN: This is a warning that should not appear in silent test output|nWARNING |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib warning that should not appear in silent test output|nQDEBUG: This is a debug message that should not appear in silent test output|nQSYSTEM: This is a critical message that should not appear in silent test output|nQINFO: This is an info message that should not appear in silent test output|nINFO |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib info message that should not appear in silent test output|nQFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Blacklisted'] -##teamcity[testFinished name='messages()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Blacklisted'] ##teamcity[testSuiteFinished name='tst_Blacklisted' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index 89085e2f56..62a2d11ed2 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -11,16 +11,6 @@ BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS BPASS : tst_Blacklisted::xfail() BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -QWARN : tst_Blacklisted::messages() This is a warning that should not appear in silent test output -WARNING: tst_Blacklisted::messages() This is an internal testlib warning that should not appear in silent test output - Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -QDEBUG : tst_Blacklisted::messages() This is a debug message that should not appear in silent test output -QSYSTEM: tst_Blacklisted::messages() This is a critical message that should not appear in silent test output -QINFO : tst_Blacklisted::messages() This is an info message that should not appear in silent test output -INFO : tst_Blacklisted::messages() This is an internal testlib info message that should not appear in silent test output - Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -QFATAL : tst_Blacklisted::messages() This is a fatal error message that should still appear in silent test output -BFAIL : tst_Blacklisted::messages() Received a fatal error. - Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -Totals: 1 passed, 0 failed, 1 skipped, 6 blacklisted, 0ms +PASS : tst_Blacklisted::cleanupTestCase() +Totals: 2 passed, 0 failed, 1 skipped, 5 blacklisted, 0ms ********* Finished testing of tst_Blacklisted ********* diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index bfd42f4214..c23ab177f6 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -38,31 +38,8 @@ - - - - - - - - - - - - - - - - - - - - - - - - - + +