AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

qtestlib: exit with error if a test is invoked with unknown data tag

Browse Source 
Previously trying to execute a test function with an unknown data tag
would print an error message but exit with 0.

This patch stores a test failure, and continues trying to execute the
rest of the command line arguments, if any. In the end the process exits
with the usual exit code (number of failed tests) which is now !=0.

Pick-to: 6.4 6.3 6.2
Fixes: QTBUG-24240
Change-Id: Id4d422035f173e01e77ca88028dfd94dc0f9085c
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Dimitrios Apostolou 2022-05-10 16:32:29 +02:00
parent 8a96c8a22c
commit d23dcc0542
2 changed files with 74 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/testlib/qtestcase.cpp
View File

@ -1292,6 +1292,26 @@ public:
#endif // QT_CONFIG(thread)
static void printUnknownDataTagError(QLatin1StringView name, QLatin1StringView tag,
const QTestTable &lTable, const QTestTable &gTable)
{
fprintf(stderr, "Unknown testdata for function %s(): '%s'\n", name.constData(), tag.data());
const int localDataCount = lTable.dataCount();
if (localDataCount) {
fputs("Available test-specific data tags:\n", stderr);
for (int i = 0; i < localDataCount; ++i)
fprintf(stderr, "\t%s\n", lTable.testData(i)->dataTag());
}
const int globalDataCount = gTable.dataCount();
if (globalDataCount) {
fputs("Available global data tags:\n", stderr);
for (int i = 0; i < globalDataCount; ++i)
fprintf(stderr, "\t%s\n", gTable.testData(i)->dataTag());
}
if (localDataCount == 0 && globalDataCount == 0)
fputs("Function has no data tags\n", stderr);
}
/*!
\internal
@ -1352,13 +1372,6 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
return dataCount ? table.testData(index)->dataTag() : nullptr;
};
// Data tag requested but none available?
if (!tag.isEmpty() && !dataCount && !globalDataCount) {
fprintf(stderr, "Unknown test data tag for function %s(): '%s'\n"
"Function has no testdata.\n", name.constData(), tag.data());
return false;
}
/* For each entry in this test's data table, do: */
do {
QTestResult::setSkipCurrentTest(false);
@ -1366,7 +1379,6 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)),
QLatin1StringView(globalDataTag(curGlobalDataIndex)))) {
foundFunction = true;
QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex),
globalDataTag(curGlobalDataIndex));
@ -1391,20 +1403,9 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
} while (curGlobalDataIndex < globalDataCount);
if (!tag.isEmpty() && !foundFunction) {
fprintf(stderr, "Unknown testdata for function %s(): '%s'\n", name.constData(), tag.data());
if (table.dataCount()) {
fputs("Available test-specific data tags:\n", stderr);
for (int i = 0; i < table.dataCount(); ++i)
fprintf(stderr, "\t%s\n", table.testData(i)->dataTag());
}
if (globalDataCount) {
fputs("Available global data tags:\n", stderr);
for (int i = 0; i < globalDataCount; ++i)
fprintf(stderr, "\t%s\n", gTable->testData(i)->dataTag());
}
return false;
printUnknownDataTagError(QLatin1StringView(name), tag, table, *gTable);
QTestResult::addFailure(qPrintable("Data tag not found: %1"_L1.arg(tag)));
}
QTestResult::finishedCurrentTestFunction();
QTestResult::setSkipCurrentTest(false);
QTestResult::setBlacklistCurrentTest(false);

52
tests/auto/testlib/selftests/tst_selftests.cpp
View File

@ -1195,6 +1195,58 @@ SCENARIO("Test output of the loggers is as expected")
}
}
struct TestCase {
int expectedExitCode;
const char *cmdline;
};
SCENARIO("Exit code is as expected")
{
// Listing of test command lines and expected exit codes
// NOTE: Use at least 2 spaces to separate arguments because some contain a space themselves.
const struct TestCase testCases[] = {
// 'pass' is a test with no data tags at all
{ 0, "pass testNumber1" },
{ 1, "pass unknownFunction" },
{ 1, "pass testNumber1:blah" },
{ 1, "pass testNumber1:blah:blue" },
// 'counting' is a test that has only local data tags
{ 0, "counting testPassPass" },
{ 0, "counting testPassPass:row 1" },
{ 1, "counting testPassPass:blah" },
{ 1, "counting testPassPass:blah:row 1" },
{ 1, "counting testPassPass:blah:blue" },
// 'globaldata' is a test with global and local data tags
{ 0, "globaldata testGlobal" },
{ 0, "globaldata testGlobal:global=true" },
{ 0, "globaldata testGlobal:local=true" },
{ 0, "globaldata testGlobal:global=true:local=true" },
{ 1, "globaldata testGlobal:local=true:global=true" },
{ 1, "globaldata testGlobal:global=true:blah" },
{ 1, "globaldata testGlobal:blah:local=true" },
{ 1, "globaldata testGlobal:blah:global=true" },
{ 1, "globaldata testGlobal:blah" },
{ 1, "globaldata testGlobal:blah:blue" },
// Passing multiple testcase:data on the command line
{ 0, "globaldata testGlobal:global=true skipSingle:global=true:local=true" },
{ 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" },
{ 1, "globaldata testGlobal:global=true skipSingle:blah" },
{ 2, "globaldata testGlobal:blah skipSingle:blue" },
};
size_t n_testCases = sizeof(testCases) / sizeof(*testCases);
for (size_t i = 0; i < n_testCases; i++) {
GIVEN("The command line: " << testCases[i].cmdline) {
const QStringList cmdSplit = QString(testCases[i].cmdline)
.split(QRegularExpression(" +")); // at least 2 spaces
const QString test = cmdSplit[0];
const QStringList args = cmdSplit.sliced(1);
auto runResult = runTestProcess(test, args);
REQUIRE(runResult.exitCode == testCases[i].expectedExitCode);
}
}
}
// ----------------------- Entrypoint -----------------------
int main(int argc, char **argv)