Add QTest::failOnWarning

This solves the long-standing problem of not being able to easily
fail a test when a certain warning is output.

[ChangeLog][QtTest] Added QTest::failOnWarning. When called in a test
function, any warning that matches the given pattern will cause a test
failure. The test will continue execution when a failure is added.
All patterns are cleared at the end of each test function.

Fixes: QTBUG-70029
Change-Id: I5763f8d4acf1cee8178be43a503619fbfb0f4f36
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Mitch Curtis 2021-10-08 13:58:27 +02:00
parent 1edf153a6b
commit efb283fb7f
12 changed files with 692 additions and 7 deletions

View File

@ -2127,6 +2127,68 @@ void QTest::ignoreMessage(QtMsgType type, const QRegularExpression &messagePatte
}
#endif // QT_CONFIG(regularexpression)
/*!
\since 6.3
\overload failOnWarning()
Appends a test failure to the test log if the \a message is output.
\sa failOnWarning()
*/
void QTest::failOnWarning(const char *message)
{
return QTestLog::failOnWarning(message);
}
#if QT_CONFIG(regularexpression)
/*!
\since 6.3
Appends a test failure to the test log for each warning that matches
\a messagePattern.
The test function will continue execution when a failure is added. To abort
the test instead, you can check \l currentTestFailed() and return early if
it's \c true.
For each warning, the first pattern that matches will cause a failure,
and the remaining patterns will be ignored.
All patterns are cleared at the end of each test function.
\code
void FileTest::loadFiles()
{
QTest::failOnWarning(QRegularExpression("^Failed to load"));
// Each of these will cause a test failure:
qWarning() << "Failed to load image";
qWarning() << "Failed to load video";
}
\endcode
To fail every test that triggers a given warning, pass a suitable regular
expression to this function in \l {Creating a Test}{init()}:
\code
void FileTest::init()
{
QTest::failOnWarning(QRegularExpression(".?"));
}
\endcode
\note \l ignoreMessage() takes precedence over this function, so any
warnings that match a pattern given to both \c ignoreMessage() and
\c failOnWarning() will be ignored.
\sa {Qt Test Environment Variables}{QTEST_FATAL_FAIL}
*/
void QTest::failOnWarning(const QRegularExpression &messagePattern)
{
QTestLog::failOnWarning(messagePattern);
}
#endif // QT_CONFIG(regularexpression)
/*! \internal
*/

View File

@ -331,6 +331,10 @@ namespace QTest
#if QT_CONFIG(regularexpression)
Q_TESTLIB_EXPORT void ignoreMessage(QtMsgType type, const QRegularExpression &messagePattern);
#endif
Q_TESTLIB_EXPORT void failOnWarning(const char *message);
#if QT_CONFIG(regularexpression)
Q_TESTLIB_EXPORT void failOnWarning(const QRegularExpression &messagePattern);
#endif
#if QT_CONFIG(temporaryfile)
Q_TESTLIB_EXPORT QSharedPointer<QTemporaryDir> qExtractTestData(const QString &dirName);

View File

@ -68,6 +68,7 @@
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <vector>
#include <vector>
#include <memory>
@ -171,6 +172,8 @@ namespace QTest {
static IgnoreResultList *ignoreResultList = nullptr;
static std::vector<QVariant> failOnWarningList;
Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)
static int verbosity = 0;
@ -205,6 +208,32 @@ namespace QTest {
return false;
}
static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
{
// failOnWarnings can be called multiple times per test function, so let
// each call cause a failure if required.
for (const auto &pattern : failOnWarningList) {
if (pattern.metaType() == QMetaType::fromType<QString>()) {
if (message != pattern.toString())
continue;
}
#if QT_CONFIG(regularexpression)
else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
if (!message.contains(pattern.toRegularExpression()))
continue;
}
#endif
const size_t maxMsgLen = 1024;
char msg[maxMsgLen] = {'\0'};
qsnprintf(msg, maxMsgLen, "Received a warning that resulted in a failure:\n%s",
qPrintable(message));
QTestResult::addFailure(msg, context.file, context.line);
return true;
}
return false;
}
static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
{
static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
@ -220,6 +249,9 @@ namespace QTest {
return;
}
if (type == QtWarningMsg && handleFailOnWarning(context, message))
return;
if (type != QtFatalMsg) {
if (counter.loadRelaxed() <= 0)
return;
@ -313,6 +345,11 @@ void QTestLog::clearIgnoreMessages()
QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
}
void QTestLog::clearFailOnWarnings()
{
QTest::failOnWarningList.clear();
}
void QTestLog::addPass(const char *msg)
{
if (printAvailableTags)
@ -546,6 +583,20 @@ void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expressio
}
#endif // QT_CONFIG(regularexpression)
void QTestLog::failOnWarning(const char *msg)
{
QTest::failOnWarningList.push_back(QString::fromUtf8(msg));
}
#if QT_CONFIG(regularexpression)
void QTestLog::failOnWarning(const QRegularExpression &expression)
{
QTEST_ASSERT(expression.isValid());
QTest::failOnWarningList.push_back(QVariant::fromValue(expression));
}
#endif // QT_CONFIG(regularexpression)
void QTestLog::setMaxWarnings(int m)
{
QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;

View File

@ -104,10 +104,15 @@ public:
static void ignoreMessage(QtMsgType type, const char *msg);
#ifndef QT_NO_REGULAREXPRESSION
static void ignoreMessage(QtMsgType type, const QRegularExpression &expression);
#endif
static void failOnWarning(const char *msg);
#ifndef QT_NO_REGULAREXPRESSION
static void failOnWarning(const QRegularExpression &expression);
#endif
static int unhandledIgnoreMessages();
static void printUnhandledIgnoreMessages();
static void clearIgnoreMessages();
static void clearFailOnWarnings();
static void warn(const char *msg, const char *file, int line);
static void info(const char *msg, const char *file, int line);

View File

@ -191,6 +191,7 @@ void QTestResult::finishedCurrentTestData()
addFailure("Not all expected messages were received");
}
QTestLog::clearIgnoreMessages();
QTestLog::clearFailOnWarnings();
}
/*!

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="4" errors="0" skipped="0" time="@TEST_DURATION@">
<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="1" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -49,5 +49,63 @@
<![CDATA[Did not receive message: "Warning1"]]>
</system-out>
</testcase>
<testcase name="testFailOnWarnings" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[Ran out of cabbage!]]>
</failure>
<system-out>
<![CDATA[Ran out of tortillas!]]>
<![CDATA[Ran out of oil!]]>
<![CDATA[Running low on toothpaste!]]>
<![CDATA[Running low on toothpaste!]]>
</system-out>
<system-err>
<![CDATA[Ran out of space!]]>
<![CDATA[nope]]>
<![CDATA[Running low on flour!]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsCleared" classname="tst_Warnings" time="@TEST_DURATION@">
<system-err>
<![CDATA[Ran out of muffins!]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsWithData(warning1)" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[warning1]]>
</failure>
<system-err>
<![CDATA[warning2]]>
<![CDATA[warning3]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsWithData(warning2)" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[warning2]]>
</failure>
<system-err>
<![CDATA[warning1]]>
<![CDATA[warning3]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsWithData(warning3)" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[warning3]]>
</failure>
<system-err>
<![CDATA[warning1]]>
<![CDATA[warning2]]>
</system-err>
</testcase>
<testcase name="testFailOnWarningsFailInHelper" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="This failure message should be printed but not cause the test to abort"/>
</testcase>
<testcase name="testFailOnWarningsThenSkip" classname="tst_Warnings" time="@TEST_DURATION@">
<failure type="fail" message="Received a warning that resulted in a failure:">
<![CDATA[Ran out of cabbage!]]>
</failure>
<skipped message="My cabbage! :("/>
</testcase>
<testcase name="testFailOnWarningsAndIgnoreWarnings" classname="tst_Warnings" time="@TEST_DURATION@"/>
<testcase name="cleanupTestCase" classname="tst_Warnings" time="@TEST_DURATION@"/>
</testsuite>

View File

@ -89,6 +89,118 @@
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarnings">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of space!]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Ran out of tortillas!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Ran out of oil!]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[nope]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of biscuits!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Running low on toothpaste!]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Running low on flour!]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsCleared">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of muffins!]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsWithData">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning1]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning2]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning3]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsFailInHelper">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsThenSkip">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>

View File

@ -35,8 +35,103 @@ not ok 6 - testMissingWarningsWithData(second row)
---
# Not all expected messages were received
...
ok 7 - cleanupTestCase()
1..7
# tests 7
# pass 3
# fail 4
# Ran out of space!
not ok 7 - testFailOnWarnings()
---
# Received a warning that resulted in a failure:
Ran out of cabbage!
at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# Ran out of tortillas!
# Ran out of oil!
# nope
not ok 8 - testFailOnWarnings()
---
# Received a warning that resulted in a failure:
Ran out of biscuits!
at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 9 - testFailOnWarnings()
---
# Received a warning that resulted in a failure:
Running low on toothpaste!
at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# Running low on flour!
# Running low on toothpaste!
# Running low on toothpaste!
# Ran out of muffins!
ok 10 - testFailOnWarningsCleared()
not ok 11 - testFailOnWarningsWithData(warning1)
---
# Received a warning that resulted in a failure:
warning1
at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# warning2
# warning3
# warning1
not ok 12 - testFailOnWarningsWithData(warning2)
---
# Received a warning that resulted in a failure:
warning2
at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
# warning3
# warning1
# warning2
not ok 13 - testFailOnWarningsWithData(warning3)
---
# Received a warning that resulted in a failure:
warning3
at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 14 - testFailOnWarningsFailInHelper()
---
# This failure message should be printed but not cause the test to abort
at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 15 - testFailOnWarningsFailInHelper()
---
# Received a warning that resulted in a failure:
Ran out of cabbage!
at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 16 - testFailOnWarningsFailInHelper()
---
# My cabbage! :(
at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
not ok 17 - testFailOnWarningsThenSkip()
---
# Received a warning that resulted in a failure:
Ran out of cabbage!
at: tst_Warnings::testFailOnWarningsThenSkip() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
line: 0
...
ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :(
ok 19 - testFailOnWarningsAndIgnoreWarnings()
ok 20 - cleanupTestCase()
1..20
# tests 20
# pass 5
# fail 14

View File

@ -20,6 +20,42 @@
##teamcity[testFailed name='testMissingWarningsWithData(second row)' message='Failure!' details='Not all expected messages were received' flowId='tst_Warnings']
##teamcity[testStdOut name='testMissingWarningsWithData(second row)' out='INFO: Did not receive message: "Warning0"|nINFO: Did not receive message: "Warning1"' flowId='tst_Warnings']
##teamcity[testFinished name='testMissingWarningsWithData(second row)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarnings()' out='QWARN: Ran out of space!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of biscuits!' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarnings()' out='QDEBUG: Ran out of tortillas!|nQINFO: Ran out of oil!|nQWARN: nope' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRunning low on toothpaste!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsCleared()' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarningsCleared()' out='QWARN: Running low on flour!|nQDEBUG: Running low on toothpaste!|nQINFO: Running low on toothpaste!|nQWARN: Ran out of muffins!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsCleared()' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsWithData(warning1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning1' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsWithData(warning2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning2' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarningsWithData(warning2)' out='QWARN: warning2|nQWARN: warning3|nQWARN: warning1' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsWithData(warning3)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning3' flowId='tst_Warnings']
##teamcity[testStdOut name='testFailOnWarningsWithData(warning3)' out='QWARN: warning3|nQWARN: warning1|nQWARN: warning2' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='This failure message should be printed but not cause the test to abort' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='My cabbage! :(' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsThenSkip()' flowId='tst_Warnings']
##teamcity[testFailed name='testFailOnWarningsThenSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsThenSkip()' flowId='tst_Warnings']
##teamcity[testIgnored name='testFailOnWarningsThenSkip()' message='My cabbage! :( |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' flowId='tst_Warnings']
##teamcity[testStarted name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings']
##teamcity[testFinished name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Warnings']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Warnings']
##teamcity[testSuiteFinished name='tst_Warnings' flowId='tst_Warnings']

View File

@ -23,6 +23,52 @@ FAIL! : tst_Warnings::testMissingWarningsWithData(first row) Not all expected m
INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning0"
INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning1"
FAIL! : tst_Warnings::testMissingWarningsWithData(second row) Not all expected messages were received
QWARN : tst_Warnings::testFailOnWarnings() Ran out of space!
FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
Ran out of cabbage!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QDEBUG : tst_Warnings::testFailOnWarnings() Ran out of tortillas!
QINFO : tst_Warnings::testFailOnWarnings() Ran out of oil!
QWARN : tst_Warnings::testFailOnWarnings() nope
FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
Ran out of biscuits!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
Running low on toothpaste!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QWARN : tst_Warnings::testFailOnWarnings() Running low on flour!
QDEBUG : tst_Warnings::testFailOnWarnings() Running low on toothpaste!
QINFO : tst_Warnings::testFailOnWarnings() Running low on toothpaste!
QWARN : tst_Warnings::testFailOnWarningsCleared() Ran out of muffins!
PASS : tst_Warnings::testFailOnWarningsCleared()
FAIL! : tst_Warnings::testFailOnWarningsWithData(warning1) Received a warning that resulted in a failure:
warning1
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QWARN : tst_Warnings::testFailOnWarningsWithData(warning1) warning2
QWARN : tst_Warnings::testFailOnWarningsWithData(warning1) warning3
QWARN : tst_Warnings::testFailOnWarningsWithData(warning2) warning1
FAIL! : tst_Warnings::testFailOnWarningsWithData(warning2) Received a warning that resulted in a failure:
warning2
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
QWARN : tst_Warnings::testFailOnWarningsWithData(warning2) warning3
QWARN : tst_Warnings::testFailOnWarningsWithData(warning3) warning1
QWARN : tst_Warnings::testFailOnWarningsWithData(warning3) warning2
FAIL! : tst_Warnings::testFailOnWarningsWithData(warning3) Received a warning that resulted in a failure:
warning3
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() This failure message should be printed but not cause the test to abort
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() Received a warning that resulted in a failure:
Ran out of cabbage!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() My cabbage! :(
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
FAIL! : tst_Warnings::testFailOnWarningsThenSkip() Received a warning that resulted in a failure:
Ran out of cabbage!
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
SKIP : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :(
Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
PASS : tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
PASS : tst_Warnings::cleanupTestCase()
Totals: 3 passed, 4 failed, 0 skipped, 0 blacklisted, 0ms
Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_Warnings *********

View File

@ -91,6 +91,118 @@
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarnings">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of space!]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Ran out of tortillas!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Ran out of oil!]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[nope]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of biscuits!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Running low on toothpaste!]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Running low on flour!]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Message type="qinfo" file="" line="0">
<Description><![CDATA[Running low on toothpaste!]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsCleared">
<Message type="qwarn" file="" line="0">
<Description><![CDATA[Ran out of muffins!]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsWithData">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning1]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning1]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning2]]></Description>
</Incident>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning2]]></DataTag>
<Description><![CDATA[warning3]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning1]]></Description>
</Message>
<Message type="qwarn" file="" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[warning2]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<DataTag><![CDATA[warning3]]></DataTag>
<Description><![CDATA[Received a warning that resulted in a failure:
warning3]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsFailInHelper">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsThenSkip">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[Received a warning that resulted in a failure:
Ran out of cabbage!]]></Description>
</Incident>
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
<Description><![CDATA[My cabbage! :(]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>

View File

@ -40,6 +40,16 @@ private slots:
void testMissingWarningsRegularExpression();
void testMissingWarningsWithData_data();
void testMissingWarningsWithData();
void testFailOnWarnings();
void testFailOnWarningsCleared();
#if QT_CONFIG(regularexpression)
void testFailOnWarningsWithData_data();
void testFailOnWarningsWithData();
void testFailOnWarningsFailInHelper();
void testFailOnWarningsThenSkip();
#endif
void testFailOnWarningsAndIgnoreWarnings();
};
void tst_Warnings::testWarnings()
@ -124,6 +134,99 @@ void tst_Warnings::testMissingWarningsWithData()
qWarning("Warning2");
}
void tst_Warnings::testFailOnWarnings()
{
// failOnWarnings() wasn't called yet; shouldn't fail;
qWarning("Ran out of space!");
#if QT_CONFIG(regularexpression)
const auto warnRegex = QRegularExpression("Ran out of .*!");
QTest::failOnWarning(warnRegex);
// Should now fail.
qWarning("Ran out of cabbage!");
// Should not fail; none of these are warnings.
qDebug("Ran out of tortillas!");
qInfo("Ran out of oil!");
// Should not fail; regex doesn't match.
qWarning("nope");
// Should fail; matches regex.
qWarning("Ran out of biscuits!");
#endif // QT_CONFIG(regularexpression)
QTest::failOnWarning("Running low on toothpaste!");
// Should fail; strings match.
qWarning("Running low on toothpaste!");
// Shouldn't fail; strings don't match.
qWarning("Running low on flour!");
// Should not fail; none of these are warnings.
qDebug("Running low on toothpaste!");
qInfo("Running low on toothpaste!");
}
void tst_Warnings::testFailOnWarningsCleared()
{
// The patterns passed to failOnWarnings() should be cleared at the end of
// each test function, so this shouldn't fail because of the failOnWarning() call in the previous function.
// Note that this test always needs to come after testFailOnWarnings for it to work.
qWarning("Ran out of muffins!");
}
#if QT_CONFIG(regularexpression)
void tst_Warnings::testFailOnWarningsWithData_data()
{
// The warning message that should cause a failure.
QTest::addColumn<QString>("warningMessage");
QTest::newRow("warning1") << "warning1";
QTest::newRow("warning2") << "warning2";
QTest::newRow("warning3") << "warning3";
}
void tst_Warnings::testFailOnWarningsWithData()
{
QFETCH(QString, warningMessage);
QTest::failOnWarning(QRegularExpression(warningMessage));
// Only one of these should fail, depending on warningMessage.
qWarning("warning1");
qWarning("warning2");
qWarning("warning3");
}
void tst_Warnings::testFailOnWarningsFailInHelper()
{
[](){ QFAIL("This failure message should be printed but not cause the test to abort"); }();
const auto warnRegex = QRegularExpression("Ran out of .*!");
QTest::failOnWarning(warnRegex);
qWarning("Ran out of cabbage!");
QFAIL("My cabbage! :(");
}
void tst_Warnings::testFailOnWarningsThenSkip()
{
const auto warnRegex = QRegularExpression("Ran out of .*!");
QTest::failOnWarning(warnRegex);
qWarning("Ran out of cabbage!");
QSKIP("My cabbage! :(");
}
#endif // QT_CONFIG(regularexpression)
void tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
{
const auto warningStr = "Running low on toothpaste!";
QTest::failOnWarning(warningStr);
QTest::ignoreMessage(QtWarningMsg, warningStr);
// Shouldn't fail; we ignored it.
qWarning(warningStr);
}
QTEST_MAIN(tst_Warnings)
#include "tst_warnings.moc"