AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove QTest::SkipMode from qtestlib API.

Browse Source 
When the SkipAll mode is used, tests only report a SKIP for the first
line of test data and subsequent lines are not reported at all.  This
behaviour makes it impossible for anything post-processing test results
to accurately report test pass- and run- rates because they cannot see
how many lines of test data were skipped.

This commit removes SkipMode.  QSKIPs in regular test functions and data
functions are treated the same as SkipSingle, so that every skipped line
of local or global test data is reported in the test log.  QSKIPs
elsewhere are treated the same as SkipAll -- skipping in init() causes
the next test function to be skipped entirely, and skipping in
initTestCase() or initTestCase_data() causes all test functions to be
skipped.

This commit only changes qtestlib and the selftests. A further commit
will change the autotests to remove the SkipMode parameter from QSKIP
calls.

Note that the change in expected output for the globaldata selftest is
deliberate, as the QSKIP in the skipLocal test function has effectively
changed from SkipAll to SkipSingle.

Task-number: QTBUG-21851, QTBUG-21652
Change-Id: I7b1c53fe7ca9dde032810b789d967e2a402bbe5d
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
Reviewed-by: Alex <alex.blasche@nokia.com>
This commit is contained in:
Jason McDonald 2011-10-18 14:53:23 +10:00 committed by Qt by Nokia
parent 296bc32841
commit 016cd01846
22 changed files with 75 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/src/snippets/code/src_qtestlib_qtestcase.cpp
View File

@ -100,7 +100,7 @@ QTEST(QString("hello").toUpper(), "myString");
//! [8]
if (!QSqlDatabase::drivers().contains("SQLITE"))
QSKIP("This test requires the SQLITE database driver", SkipAll);
QSKIP("This test requires the SQLITE database driver");
//! [8]

1
src/testlib/qtest_global.h
View File

@ -77,7 +77,6 @@ QT_MODULE(Test)
namespace QTest
{
enum SkipMode { SkipSingle = 1, SkipAll = 2 };
enum TestFailMode { Abort = 1, Continue = 2 };
int Q_TESTLIB_EXPORT qt_snprintf(char *str, int size, const char *format, ...);

47
src/testlib/qtestcase.cpp
View File

@ -267,23 +267,24 @@ QT_BEGIN_NAMESPACE
\sa QCOMPARE()
*/
/*! \macro QSKIP(description, mode)
/*! \macro QSKIP(description)
\relates QTest
The QSKIP() macro stops execution of the test without adding a failure to the
test log. You can use it to skip tests that wouldn't make sense in the current
configuration. The text \a description is appended to the test log and should
contain an explanation why the test couldn't be executed. \a mode is a QTest::SkipMode
and describes whether to proceed with the rest of the test data or not.
contain an explanation of why the test couldn't be executed.
If the test is data-driven, each call to QSKIP() will skip only the current row,
so an unconditional call to QSKIP will produce one skip message in the test log
for each row of test data.
\bold {Note:} This macro can only be used in a test function that is invoked
by the test framework.
Example:
\snippet doc/src/snippets/code/src_qtestlib_qtestcase.cpp 8
\sa QTest::SkipMode
*/
/*! \macro QEXPECT_FAIL(dataIndex, comment, mode)
@ -415,22 +416,6 @@ QT_BEGIN_NAMESPACE
{Chapter 5: Writing a Benchmark}{Writing a Benchmark}
*/
/*! \enum QTest::SkipMode
This enum describes the modes for skipping tests during execution
of the test data.
\value SkipSingle Skips the current entry in the test table; continues
execution of all the other entries in the table.
\value SkipAll Skips all the entries in the test table; the test won't
be executed further.
\sa QSKIP()
*/
/*! \enum QTest::TestFailMode
This enum describes the modes for handling an expected failure of the
@ -1469,20 +1454,12 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
QTestResult::setCurrentTestLocation(QTestResult::DataFunc);
QTest::qt_snprintf(member, 512, "%s_data()", slot);
invokeMethod(QTest::currentTestObject, member);
// if we encounter a SkipAll in the _data slot, we skip the whole
// testfunction, no matter how much global data exists
if (QTestResult::skipCurrentTest()) {
QTestResult::setCurrentGlobalTestData(0);
break;
}
}
bool foundFunction = false;
if (!QTestResult::skipCurrentTest()) {
int curDataIndex = 0;
const int dataCount = table.dataCount();
QTestResult::setSkipCurrentTest(false);
// Data tag requested but none available?
if (data && !dataCount) {
@ -1498,6 +1475,7 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
/* For each entry in the data table, do: */
do {
QTestResult::setSkipCurrentTest(false);
if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) {
foundFunction = true;
QTestDataSetter s(curDataIndex >= dataCount ? static_cast<QTestData *>(0)
@ -1505,9 +1483,6 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
qInvokeTestMethodDataEntry(slot);
if (QTestResult::skipCurrentTest())
// check whether SkipAll was requested
break;
if (data)
break;
}
@ -1954,15 +1929,13 @@ bool QTest::qVerify(bool statement, const char *statementStr, const char *descri
return QTestResult::verify(statement, statementStr, description, file, line);
}
/*! \fn void QTest::qSkip(const char *message, SkipMode mode, const char *file, int line)
/*! \fn void QTest::qSkip(const char *message, const char *file, int line)
\internal
*/
void QTest::qSkip(const char *message, QTest::SkipMode mode,
const char *file, int line)
void QTest::qSkip(const char *message, const char *file, int line)
{
QTestResult::addSkip(message, file, line);
if (mode == QTest::SkipAll)
QTestResult::setSkipCurrentTest(true);
QTestResult::setSkipCurrentTest(true);
}
/*! \fn bool QTest::qExpectFail(const char *dataIndex, const char *comment, TestFailMode mode, const char *file, int line)

11
src/testlib/qtestcase.h
View File

@ -47,6 +47,8 @@
#include <QtCore/qnamespace.h>
#include <QtCore/qmetatype.h>
#include <string.h>
QT_BEGIN_HEADER
QT_BEGIN_NAMESPACE
@ -110,9 +112,12 @@ do { \
QCOMPARE(__expr, __expected); \
} while (0)
#define QSKIP(statement, mode) \
#define QSKIP(statement, ...) \
do {\
QTest::qSkip(statement, QTest::mode, __FILE__, __LINE__);\
if (strcmp(#__VA_ARGS__, "") != 0)\
QTest::qWarn("The two argument version of QSKIP is deprecated and will be removed soon. "\
"Please update this test by removing the second parameter.", __FILE__, __LINE__);\
QTest::qSkip(statement, __FILE__, __LINE__);\
return;\
} while (0)
@ -162,7 +167,7 @@ namespace QTest
Q_TESTLIB_EXPORT bool qVerify(bool statement, const char *statementStr, const char *description,
const char *file, int line);
Q_TESTLIB_EXPORT void qFail(const char *statementStr, const char *file, int line);
Q_TESTLIB_EXPORT void qSkip(const char *message, SkipMode mode, const char *file, int line);
Q_TESTLIB_EXPORT void qSkip(const char *message, const char *file, int line);
Q_TESTLIB_EXPORT bool qExpectFail(const char *dataIndex, const char *comment, TestFailMode mode,
const char *file, int line);
Q_TESTLIB_EXPORT void qWarn(const char *message, const char *file = 0, int line = 0);

2
tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
View File

@ -76,7 +76,7 @@ void tst_BenchlibCallgrind::failInChildProcess()
void tst_BenchlibCallgrind::twoHundredMillionInstructions()
{
#if !defined(__GNUC__) || !defined(__i386)
QSKIP("This test is only defined for gcc and x86.", SkipAll);
QSKIP("This test is only defined for gcc and x86.");
#else
QBENCHMARK {
__asm__ __volatile__(

2
tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
View File

@ -56,7 +56,7 @@ private slots:
void tst_BenchlibTickCounter::threeBillionTicks()
{
#ifndef HAVE_TICK_COUNTER
QSKIP("Tick counter not available on this platform", SkipAll);
QSKIP("Tick counter not available on this platform");
#else
QBENCHMARK {
CycleCounterTicks start = getticks();

12
tests/auto/testlib/selftests/expected_globaldata.lightxml
View File

@ -94,6 +94,18 @@
<DataTag><![CDATA[1:local 1]]></DataTag>
<Description><![CDATA[cleanup skipLocal local 1 ]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[1:local 2]]></DataTag>
<Description><![CDATA[init skipLocal local 2 ]]></Description>
</Message>
<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp" line="149">
<DataTag><![CDATA[1:local 2]]></DataTag>
<Description><![CDATA[skipping]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[1:local 2]]></DataTag>
<Description><![CDATA[cleanup skipLocal local 2 ]]></Description>
</Message>
</TestFunction>
<TestFunction name="skipSingle">
<Message type="qdebug" file="" line="0">

6
tests/auto/testlib/selftests/expected_globaldata.txt
View File

@ -25,6 +25,10 @@ QDEBUG : tst_globaldata::skipLocal(1:local 1) init skipLocal local 1
SKIP : tst_globaldata::skipLocal(1:local 1) skipping
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)]
QDEBUG : tst_globaldata::skipLocal(1:local 1) cleanup skipLocal local 1
QDEBUG : tst_globaldata::skipLocal(1:local 2) init skipLocal local 2
SKIP : tst_globaldata::skipLocal(1:local 2) skipping
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)]
QDEBUG : tst_globaldata::skipLocal(1:local 2) cleanup skipLocal local 2
QDEBUG : tst_globaldata::skipSingle(1:local 1) init skipSingle local 1
QDEBUG : tst_globaldata::skipSingle(1:local 1) global: false local: false
QDEBUG : tst_globaldata::skipSingle(1:local 1) cleanup skipSingle local 1
@ -41,5 +45,5 @@ QDEBUG : tst_globaldata::skipSingle(2:local 2) cleanup skipSingle local 2
PASS : tst_globaldata::skipSingle()
QDEBUG : tst_globaldata::cleanupTestCase() cleanupTestCase cleanupTestCase (null)
PASS : tst_globaldata::cleanupTestCase()
Totals: 4 passed, 0 failed, 3 skipped
Totals: 4 passed, 0 failed, 4 skipped
********* Finished testing of tst_globaldata *********

12
tests/auto/testlib/selftests/expected_globaldata.xml
View File

@ -96,6 +96,18 @@
<DataTag><![CDATA[1:local 1]]></DataTag>
<Description><![CDATA[cleanup skipLocal local 1 ]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[1:local 2]]></DataTag>
<Description><![CDATA[init skipLocal local 2 ]]></Description>
</Message>
<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp" line="149">
<DataTag><![CDATA[1:local 2]]></DataTag>
<Description><![CDATA[skipping]]></Description>
</Message>
<Message type="qdebug" file="" line="0">
<DataTag><![CDATA[1:local 2]]></DataTag>
<Description><![CDATA[cleanup skipLocal local 2 ]]></Description>
</Message>
</TestFunction>
<TestFunction name="skipSingle">
<Message type="qdebug" file="" line="0">

8
tests/auto/testlib/selftests/expected_globaldata.xunitxml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite errors="34" failures="0" tests="6" name="tst_globaldata">
<testsuite errors="37" failures="0" tests="6" name="tst_globaldata">
<properties>
<property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
<property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@ -32,6 +32,9 @@
<!-- tag="1:local 1" message="init skipLocal local 1 " type="qdebug" -->
<!-- tag="1:local 1" message="skipping" type="skip" -->
<!-- tag="1:local 1" message="cleanup skipLocal local 1 " type="qdebug" -->
<!-- tag="1:local 2" message="init skipLocal local 2 " type="qdebug" -->
<!-- tag="1:local 2" message="skipping" type="skip" -->
<!-- tag="1:local 2" message="cleanup skipLocal local 2 " type="qdebug" -->
</testcase>
<testcase result="pass" name="skipSingle">
<!-- tag="1:local 1" message="init skipSingle local 1 " type="qdebug" -->
@ -72,6 +75,9 @@
<![CDATA[init skipLocal local 1 ]]>
<![CDATA[skipping]]>
<![CDATA[cleanup skipLocal local 1 ]]>
<![CDATA[init skipLocal local 2 ]]>
<![CDATA[skipping]]>
<![CDATA[cleanup skipLocal local 2 ]]>
<![CDATA[init skipSingle local 1 ]]>
<![CDATA[global: false local: false ]]>
<![CDATA[cleanup skipSingle local 1 ]]>

2
tests/auto/testlib/selftests/expected_singleskip.txt
View File

@ -2,7 +2,7 @@
Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
PASS : tst_SingleSkip::initTestCase()
SKIP : tst_SingleSkip::myTest() skipping test
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(23)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(56)]
PASS : tst_SingleSkip::cleanupTestCase()
Totals: 2 passed, 0 failed, 1 skipped
********* Finished testing of tst_SingleSkip *********

2
tests/auto/testlib/selftests/expected_skipinit.txt
View File

@ -1,7 +1,7 @@
********* Start testing of tst_SkipInit *********
Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
SKIP : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class.
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(22)]
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(55)]
PASS : tst_SkipInit::cleanupTestCase()
Totals: 1 passed, 0 failed, 1 skipped
********* Finished testing of tst_SkipInit *********

2
tests/auto/testlib/selftests/expected_skipinitdata.lightxml
View File

@ -4,6 +4,6 @@
</Environment>
<TestFunction name="initTestCase">
<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp" line="56">
<Description><![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]></Description>
<Description><![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]></Description>
</Message>
</TestFunction>

4
tests/auto/testlib/selftests/expected_skipinitdata.txt
View File

@ -1,6 +1,6 @@
********* Start testing of tst_SkipInitData *********
Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
SKIP : tst_SkipInitData::initTestCase() Skip inside initTestCase. This should skip all tests in the class.
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(23)]
SKIP : tst_SkipInitData::initTestCase() Skip inside initTestCase_data. This should skip all tests in the class.
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(56)]
Totals: 0 passed, 0 failed, 1 skipped
********* Finished testing of tst_SkipInitData *********

2
tests/auto/testlib/selftests/expected_skipinitdata.xml
View File

@ -6,7 +6,7 @@
</Environment>
<TestFunction name="initTestCase">
<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp" line="56">
<Description><![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]></Description>
<Description><![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]></Description>
</Message>
</TestFunction>
</TestCase>

4
tests/auto/testlib/selftests/expected_skipinitdata.xunitxml
View File

@ -5,9 +5,9 @@
<property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
</properties>
<testcase name="initTestCase">
<!-- message="Skip inside initTestCase. This should skip all tests in the class." type="skip" -->
<!-- message="Skip inside initTestCase_data. This should skip all tests in the class." type="skip" -->
</testcase>
<system-err>
<![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]>
<![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]>
</system-err>
</testsuite>

6
tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp
View File

@ -126,7 +126,7 @@ void tst_globaldata::skip_data()
QTest::newRow("local 1") << false;
QTest::newRow("local 2") << true;
QSKIP("skipping", SkipAll);
QSKIP("skipping");
}
void tst_globaldata::skip()
@ -140,13 +140,13 @@ void tst_globaldata::skipSingle()
QFETCH(bool, booll);
if (booli && !booll)
QSKIP("skipping", SkipSingle);
QSKIP("skipping");
qDebug() << "global:" << booli << "local:" << booll;
}
void tst_globaldata::skipLocal()
{
QSKIP("skipping", SkipAll);
QSKIP("skipping");
}
QTEST_MAIN(tst_globaldata)

2
tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp
View File

@ -53,7 +53,7 @@ private slots:
void tst_SingleSkip::myTest() const
{
QSKIP("skipping test", SkipAll);
QSKIP("skipping test");
}
QTEST_MAIN(tst_SingleSkip)

6
tests/auto/testlib/selftests/skip/tst_skip.cpp
View File

@ -65,7 +65,7 @@ void tst_Skip::test_data()
QTest::newRow("local 1") << false;
QTest::newRow("local 2") << true;
QSKIP("skipping all", SkipAll);
QSKIP("skipping all");
}
void tst_Skip::test()
@ -75,7 +75,7 @@ void tst_Skip::test()
void tst_Skip::emptytest_data()
{
QSKIP("skipping all", SkipAll);
QSKIP("skipping all");
}
void tst_Skip::emptytest()
@ -94,7 +94,7 @@ void tst_Skip::singleSkip()
{
QFETCH(bool, booll);
if (!booll)
QSKIP("skipping one", SkipSingle);
QSKIP("skipping one");
qDebug("this line should only be reached once (%s)", booll ? "true" : "false");
}

2
tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp
View File

@ -52,7 +52,7 @@ private slots:
void tst_SkipInit::initTestCase() const
{
QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll);
QSKIP("Skip inside initTestCase. This should skip all tests in the class.");
}
/*! \internal

2
tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp
View File

@ -53,7 +53,7 @@ private slots:
void tst_SkipInitData::initTestCase_data() const
{
QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll);
QSKIP("Skip inside initTestCase_data. This should skip all tests in the class.");
}
void tst_SkipInitData::initTestCase() const

2
tests/auto/testlib/selftests/xunit/tst_xunit.cpp
View File

@ -76,7 +76,7 @@ void tst_Xunit::testFunc2()
void tst_Xunit::testFunc3()
{
QSKIP("skipping this function!", SkipAll);
QSKIP("skipping this function!");
}
void tst_Xunit::testFunc4()