From d23dcc05425dcd2ab0eb621f5f2ea793a78fcc9f Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Tue, 10 May 2022 16:32:29 +0200 Subject: [PATCH] qtestlib: exit with error if a test is invoked with unknown data tag Previously trying to execute a test function with an unknown data tag would print an error message but exit with 0. This patch stores a test failure, and continues trying to execute the rest of the command line arguments, if any. In the end the process exits with the usual exit code (number of failed tests) which is now !=0. Pick-to: 6.4 6.3 6.2 Fixes: QTBUG-24240 Change-Id: Id4d422035f173e01e77ca88028dfd94dc0f9085c Reviewed-by: Edward Welbourne --- src/testlib/qtestcase.cpp | 43 +++++++-------- .../auto/testlib/selftests/tst_selftests.cpp | 52 +++++++++++++++++++ 2 files changed, 74 insertions(+), 21 deletions(-) diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index f55d4ba38f..242246d279 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -1292,6 +1292,26 @@ public: #endif // QT_CONFIG(thread) +static void printUnknownDataTagError(QLatin1StringView name, QLatin1StringView tag, + const QTestTable &lTable, const QTestTable &gTable) +{ + fprintf(stderr, "Unknown testdata for function %s(): '%s'\n", name.constData(), tag.data()); + const int localDataCount = lTable.dataCount(); + if (localDataCount) { + fputs("Available test-specific data tags:\n", stderr); + for (int i = 0; i < localDataCount; ++i) + fprintf(stderr, "\t%s\n", lTable.testData(i)->dataTag()); + } + const int globalDataCount = gTable.dataCount(); + if (globalDataCount) { + fputs("Available global data tags:\n", stderr); + for (int i = 0; i < globalDataCount; ++i) + fprintf(stderr, "\t%s\n", gTable.testData(i)->dataTag()); + } + if (localDataCount == 0 && globalDataCount == 0) + fputs("Function has no data tags\n", stderr); +} + /*! \internal @@ -1352,13 +1372,6 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo return dataCount ? table.testData(index)->dataTag() : nullptr; }; - // Data tag requested but none available? - if (!tag.isEmpty() && !dataCount && !globalDataCount) { - fprintf(stderr, "Unknown test data tag for function %s(): '%s'\n" - "Function has no testdata.\n", name.constData(), tag.data()); - return false; - } - /* For each entry in this test's data table, do: */ do { QTestResult::setSkipCurrentTest(false); @@ -1366,7 +1379,6 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)), QLatin1StringView(globalDataTag(curGlobalDataIndex)))) { foundFunction = true; - QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex), globalDataTag(curGlobalDataIndex)); @@ -1391,20 +1403,9 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo } while (curGlobalDataIndex < globalDataCount); if (!tag.isEmpty() && !foundFunction) { - fprintf(stderr, "Unknown testdata for function %s(): '%s'\n", name.constData(), tag.data()); - if (table.dataCount()) { - fputs("Available test-specific data tags:\n", stderr); - for (int i = 0; i < table.dataCount(); ++i) - fprintf(stderr, "\t%s\n", table.testData(i)->dataTag()); - } - if (globalDataCount) { - fputs("Available global data tags:\n", stderr); - for (int i = 0; i < globalDataCount; ++i) - fprintf(stderr, "\t%s\n", gTable->testData(i)->dataTag()); - } - return false; + printUnknownDataTagError(QLatin1StringView(name), tag, table, *gTable); + QTestResult::addFailure(qPrintable("Data tag not found: %1"_L1.arg(tag))); } - QTestResult::finishedCurrentTestFunction(); QTestResult::setSkipCurrentTest(false); QTestResult::setBlacklistCurrentTest(false); diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 5f67d0e25b..b70aec2c85 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -1195,6 +1195,58 @@ SCENARIO("Test output of the loggers is as expected") } } +struct TestCase { + int expectedExitCode; + const char *cmdline; +}; + +SCENARIO("Exit code is as expected") +{ + // Listing of test command lines and expected exit codes + // NOTE: Use at least 2 spaces to separate arguments because some contain a space themselves. + const struct TestCase testCases[] = { + // 'pass' is a test with no data tags at all + { 0, "pass testNumber1" }, + { 1, "pass unknownFunction" }, + { 1, "pass testNumber1:blah" }, + { 1, "pass testNumber1:blah:blue" }, + // 'counting' is a test that has only local data tags + { 0, "counting testPassPass" }, + { 0, "counting testPassPass:row 1" }, + { 1, "counting testPassPass:blah" }, + { 1, "counting testPassPass:blah:row 1" }, + { 1, "counting testPassPass:blah:blue" }, + // 'globaldata' is a test with global and local data tags + { 0, "globaldata testGlobal" }, + { 0, "globaldata testGlobal:global=true" }, + { 0, "globaldata testGlobal:local=true" }, + { 0, "globaldata testGlobal:global=true:local=true" }, + { 1, "globaldata testGlobal:local=true:global=true" }, + { 1, "globaldata testGlobal:global=true:blah" }, + { 1, "globaldata testGlobal:blah:local=true" }, + { 1, "globaldata testGlobal:blah:global=true" }, + { 1, "globaldata testGlobal:blah" }, + { 1, "globaldata testGlobal:blah:blue" }, + // Passing multiple testcase:data on the command line + { 0, "globaldata testGlobal:global=true skipSingle:global=true:local=true" }, + { 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" }, + { 1, "globaldata testGlobal:global=true skipSingle:blah" }, + { 2, "globaldata testGlobal:blah skipSingle:blue" }, + }; + + size_t n_testCases = sizeof(testCases) / sizeof(*testCases); + for (size_t i = 0; i < n_testCases; i++) { + GIVEN("The command line: " << testCases[i].cmdline) { + const QStringList cmdSplit = QString(testCases[i].cmdline) + .split(QRegularExpression(" +")); // at least 2 spaces + const QString test = cmdSplit[0]; + const QStringList args = cmdSplit.sliced(1); + auto runResult = runTestProcess(test, args); + REQUIRE(runResult.exitCode == testCases[i].expectedExitCode); + } + } +} + // ----------------------- Entrypoint ----------------------- int main(int argc, char **argv)