diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 8b9ba6d748..56959ba62d 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -1490,6 +1490,7 @@ static void qInvokeTestMethodDataEntry(char *slot) { /* Benchmarking: for each median iteration*/ + bool isBenchmark = false; int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0; QList results; @@ -1516,25 +1517,30 @@ static void qInvokeTestMethodDataEntry(char *slot) if (!invokeOk) QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__); + isBenchmark = QBenchmarkTestMethodData::current->isBenchmark(); + QTestResult::finishedCurrentTestData(); invokeMethod(QTest::currentTestObject, "cleanup()"); - QTestResult::finishedCurrentTestDataCleanup(); + + // If the test isn't a benchmark, finalize the result after cleanup() has finished. + if (!isBenchmark) + QTestResult::finishedCurrentTestDataCleanup(); // If this test method has a benchmark, repeat until all measurements are // acceptable. // The QBENCHMARK macro increases the number of iterations for each run until // this happens. - } while (invokeOk - && QBenchmarkTestMethodData::current->isBenchmark() - && QBenchmarkTestMethodData::current->resultsAccepted() == false); + } while (invokeOk && isBenchmark + && QBenchmarkTestMethodData::current->resultsAccepted() == false + && !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()); QBenchmarkTestMethodData::current->endDataRun(); - if (i > -1) // iteration -1 is the warmup iteration. - results.append(QBenchmarkTestMethodData::current->result); + if (!QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()) { + if (i > -1) // iteration -1 is the warmup iteration. + results.append(QBenchmarkTestMethodData::current->result); - if (QBenchmarkTestMethodData::current->isBenchmark() && - QBenchmarkGlobalData::current->verboseOutput) { + if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput) { if (i == -1) { QTestLog::info(qPrintable( QString::fromLatin1("warmup stage result : %1") @@ -1545,12 +1551,19 @@ static void qInvokeTestMethodDataEntry(char *slot) .arg(QBenchmarkTestMethodData::current->result.value)), 0, 0); } } - } while (QBenchmarkTestMethodData::current->isBenchmark() - && (++i < QBenchmarkGlobalData::current->adjustMedianIterationCount())); + } + } while (isBenchmark + && (++i < QBenchmarkGlobalData::current->adjustMedianIterationCount()) + && !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()); - if (QBenchmarkTestMethodData::current->isBenchmark() - && QBenchmarkTestMethodData::current->resultsAccepted()) - QTestLog::addBenchmarkResult(qMedian(results)); + // If the test is a benchmark, finalize the result after all iterations have finished. + if (isBenchmark) { + bool testPassed = !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed(); + QTestResult::finishedCurrentTestDataCleanup(); + // Only report benchmark figures if the test passed + if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted()) + QTestLog::addBenchmarkResult(qMedian(results)); + } } /*! diff --git a/tests/auto/testlib/selftests/badxml/tst_badxml.cpp b/tests/auto/testlib/selftests/badxml/tst_badxml.cpp index 1c9a0d9493..1a143e5243 100644 --- a/tests/auto/testlib/selftests/badxml/tst_badxml.cpp +++ b/tests/auto/testlib/selftests/badxml/tst_badxml.cpp @@ -106,15 +106,18 @@ void tst_BadXml::badDataTag() const QBENCHMARK { } - QFAIL("a failure"); + QFETCH(bool, shouldFail); + if (shouldFail) + QFAIL("a failure"); } void tst_BadXml::badDataTag_data() const { - QTest::addColumn("dummy"); + QTest::addColumn("shouldFail"); foreach (char const* str, badStrings()) { - QTest::newRow(str) << 0; + QTest::newRow(qPrintable(QString("fail %1").arg(str))) << true; + QTest::newRow(qPrintable(QString("pass %1").arg(str))) << false; } } diff --git a/tests/auto/testlib/selftests/benchlibcounting/benchlibcounting.pro b/tests/auto/testlib/selftests/benchlibcounting/benchlibcounting.pro new file mode 100644 index 0000000000..b495995eac --- /dev/null +++ b/tests/auto/testlib/selftests/benchlibcounting/benchlibcounting.pro @@ -0,0 +1,7 @@ +SOURCES += tst_benchlibcounting.cpp +QT = core testlib + +mac:CONFIG -= app_bundle +CONFIG -= debug_and_release_target + +TARGET = benchlibcounting diff --git a/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp b/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp new file mode 100644 index 0000000000..847bc1a81d --- /dev/null +++ b/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp @@ -0,0 +1,76 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include +#include + +class tst_BenchlibCounting : public QObject +{ + Q_OBJECT + +private slots: + void passingBenchmark(); + void skippingBenchmark(); + void failingBenchmark(); +}; + +void tst_BenchlibCounting::passingBenchmark() +{ + QBENCHMARK { + } +} + +void tst_BenchlibCounting::skippingBenchmark() +{ + QBENCHMARK { + QSKIP("This is a skipping benchmark"); + } +} + +void tst_BenchlibCounting::failingBenchmark() +{ + QBENCHMARK { + QFAIL("This is a failing benchmark"); + }; +} + +QTEST_MAIN(tst_BenchlibCounting) +#include "tst_benchlibcounting.moc" diff --git a/tests/auto/testlib/selftests/expected_badxml.lightxml b/tests/auto/testlib/selftests/expected_badxml.lightxml index a8cdf56dc1..08ba497efa 100644 --- a/tests/auto/testlib/selftests/expected_badxml.lightxml +++ b/tests/auto/testlib/selftests/expected_badxml.lightxml @@ -7,41 +7,69 @@ - text ]]]> more text]]> + text ]]]> more text]]> - - text ]]]> more text]]> + + text ]]]> more text]]> - - + text ]]]> more text]]> - - - + + text ]]]> more text]]> - + - open < tags < text]]> + - - open < tags < text]]> + + - - " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + - - " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + + + open < tags < text]]> + + + + open < tags < text]]> - + + open < tags < text]]> + + + + open < tags < text]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + diff --git a/tests/auto/testlib/selftests/expected_badxml.txt b/tests/auto/testlib/selftests/expected_badxml.txt index d4f4432066..68d333ec30 100644 --- a/tests/auto/testlib/selftests/expected_badxml.txt +++ b/tests/auto/testlib/selftests/expected_badxml.txt @@ -1,25 +1,33 @@ ********* Start testing of tst_BadXml ********* Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ PASS : tst_BadXml::initTestCase() -QDEBUG : tst_BadXml::badDataTag(end cdata ]]> text ]]> more text) a message -FAIL! : tst_BadXml::badDataTag(end cdata ]]> text ]]> more text) a failure - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(109)] -RESULT : tst_BadXml::badDataTag():"end cdata ]]> text ]]> more text": +QDEBUG : tst_BadXml::badDataTag(fail end cdata ]]> text ]]> more text) a message +FAIL! : tst_BadXml::badDataTag(fail end cdata ]]> text ]]> more text) a failure + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(111)] +QDEBUG : tst_BadXml::badDataTag(pass end cdata ]]> text ]]> more text) a message +PASS : tst_BadXml::badDataTag(pass end cdata ]]> text ]]> more text) +RESULT : tst_BadXml::badDataTag():"pass end cdata ]]> text ]]> more text": 0 events per iteration (total: 0, iterations: 1) -QDEBUG : tst_BadXml::badDataTag(quotes " text" more text) a message -FAIL! : tst_BadXml::badDataTag(quotes " text" more text) a failure - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(109)] -RESULT : tst_BadXml::badDataTag():"quotes " text" more text": +QDEBUG : tst_BadXml::badDataTag(fail quotes " text" more text) a message +FAIL! : tst_BadXml::badDataTag(fail quotes " text" more text) a failure + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(111)] +QDEBUG : tst_BadXml::badDataTag(pass quotes " text" more text) a message +PASS : tst_BadXml::badDataTag(pass quotes " text" more text) +RESULT : tst_BadXml::badDataTag():"pass quotes " text" more text": 0 events per iteration (total: 0, iterations: 1) -QDEBUG : tst_BadXml::badDataTag(xml close > open < tags < text) a message -FAIL! : tst_BadXml::badDataTag(xml close > open < tags < text) a failure - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(109)] -RESULT : tst_BadXml::badDataTag():"xml close > open < tags < text": +QDEBUG : tst_BadXml::badDataTag(fail xml close > open < tags < text) a message +FAIL! : tst_BadXml::badDataTag(fail xml close > open < tags < text) a failure + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(111)] +QDEBUG : tst_BadXml::badDataTag(pass xml close > open < tags < text) a message +PASS : tst_BadXml::badDataTag(pass xml close > open < tags < text) +RESULT : tst_BadXml::badDataTag():"pass xml close > open < tags < text": 0 events per iteration (total: 0, iterations: 1) -QDEBUG : tst_BadXml::badDataTag(all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs) a message -FAIL! : tst_BadXml::badDataTag(all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs) a failure - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(109)] -RESULT : tst_BadXml::badDataTag():"all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs": +QDEBUG : tst_BadXml::badDataTag(fail all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs) a message +FAIL! : tst_BadXml::badDataTag(fail all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs) a failure + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/badxml/tst_badxml.cpp(111)] +QDEBUG : tst_BadXml::badDataTag(pass all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs) a message +PASS : tst_BadXml::badDataTag(pass all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs) +RESULT : tst_BadXml::badDataTag():"pass all > " mixed ]]> up > " in < the ]]> hopes < of triggering "< ]]> bugs": 0 events per iteration (total: 0, iterations: 1) QDEBUG : tst_BadXml::badMessage(string 0) end cdata ]]> text ]]> more text PASS : tst_BadXml::badMessage(string 0) @@ -31,5 +39,5 @@ QDEBUG : tst_BadXml::badMessage(string 3) all > " mixed ]]> up > " in < the ]]> PASS : tst_BadXml::badMessage(string 3) FAIL! : tst_BadXml::failWithNoFile() failure message PASS : tst_BadXml::cleanupTestCase() -Totals: 6 passed, 5 failed, 0 skipped +Totals: 10 passed, 5 failed, 0 skipped ********* Finished testing of tst_BadXml ********* diff --git a/tests/auto/testlib/selftests/expected_badxml.xml b/tests/auto/testlib/selftests/expected_badxml.xml index 9f083d3917..849bc07bfc 100644 --- a/tests/auto/testlib/selftests/expected_badxml.xml +++ b/tests/auto/testlib/selftests/expected_badxml.xml @@ -9,41 +9,69 @@ - text ]]]> more text]]> + text ]]]> more text]]> - - text ]]]> more text]]> + + text ]]]> more text]]> - - + text ]]]> more text]]> - - - + + text ]]]> more text]]> - + - open < tags < text]]> + - - open < tags < text]]> + + - - " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + - - " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + + + open < tags < text]]> + + + + open < tags < text]]> - + + open < tags < text]]> + + + + open < tags < text]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + + + " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> + + diff --git a/tests/auto/testlib/selftests/expected_badxml.xunitxml b/tests/auto/testlib/selftests/expected_badxml.xunitxml index 724aed59e7..939e887a88 100644 --- a/tests/auto/testlib/selftests/expected_badxml.xunitxml +++ b/tests/auto/testlib/selftests/expected_badxml.xunitxml @@ -1,19 +1,23 @@ - + - - - - - - - - + + + + + + + + + + + + @@ -30,6 +34,10 @@ + + + + text ]]]> more text]]> open < tags < text]]> diff --git a/tests/auto/testlib/selftests/expected_benchlibcallgrind.txt b/tests/auto/testlib/selftests/expected_benchlibcallgrind.txt index 714b67fa77..13e9a39aff 100644 --- a/tests/auto/testlib/selftests/expected_benchlibcallgrind.txt +++ b/tests/auto/testlib/selftests/expected_benchlibcallgrind.txt @@ -2,9 +2,8 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ PASS : tst_BenchlibCallgrind::initTestCase() PASS : tst_BenchlibCallgrind::twoHundredMillionInstructions() -PASS : tst_BenchlibCallgrind::twoHundredMillionInstructions() RESULT : tst_BenchlibCallgrind::twoHundredMillionInstructions(): 200,000,158 instruction reads per iteration (total: 200,000,158, iterations: 1) PASS : tst_BenchlibCallgrind::cleanupTestCase() -Totals: 4 passed, 0 failed, 0 skipped +Totals: 3 passed, 0 failed, 0 skipped ********* Finished testing of tst_BenchlibCallgrind ********* diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.lightxml b/tests/auto/testlib/selftests/expected_benchlibcounting.lightxml new file mode 100644 index 0000000000..5c436a53e0 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.lightxml @@ -0,0 +1,24 @@ + + @INSERT_QT_VERSION_HERE@ + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.txt b/tests/auto/testlib/selftests/expected_benchlibcounting.txt new file mode 100644 index 0000000000..e1af40b9e0 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.txt @@ -0,0 +1,13 @@ +********* Start testing of tst_BenchlibCounting ********* +Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ +PASS : tst_BenchlibCounting::initTestCase() +PASS : tst_BenchlibCounting::passingBenchmark() +RESULT : tst_BenchlibCounting::passingBenchmark(): + 0 events per iteration (total: 0, iterations: 1) +SKIP : tst_BenchlibCounting::skippingBenchmark() This is a skipping benchmark + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp(64)] +FAIL! : tst_BenchlibCounting::failingBenchmark() This is a failing benchmark + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp(71)] +PASS : tst_BenchlibCounting::cleanupTestCase() +Totals: 3 passed, 1 failed, 1 skipped +********* Finished testing of tst_BenchlibCounting ********* diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.xml b/tests/auto/testlib/selftests/expected_benchlibcounting.xml new file mode 100644 index 0000000000..5bf71fbf8e --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.xml @@ -0,0 +1,27 @@ + + + + @INSERT_QT_VERSION_HERE@ + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.xunitxml b/tests/auto/testlib/selftests/expected_benchlibcounting.xunitxml new file mode 100644 index 0000000000..83e429aa3a --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.xunitxml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_benchliboptions.txt b/tests/auto/testlib/selftests/expected_benchliboptions.txt index 6d6d91c25e..ef9f0c5ad0 100644 --- a/tests/auto/testlib/selftests/expected_benchliboptions.txt +++ b/tests/auto/testlib/selftests/expected_benchliboptions.txt @@ -20,14 +20,8 @@ Totals: 3 passed, 0 failed, 0 skipped Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ PASS : tst_BenchlibOneHundredMinimum::initTestCase() PASS : tst_BenchlibOneHundredMinimum::threeEvents() -PASS : tst_BenchlibOneHundredMinimum::threeEvents() -PASS : tst_BenchlibOneHundredMinimum::threeEvents() -PASS : tst_BenchlibOneHundredMinimum::threeEvents() -PASS : tst_BenchlibOneHundredMinimum::threeEvents() -PASS : tst_BenchlibOneHundredMinimum::threeEvents() -PASS : tst_BenchlibOneHundredMinimum::threeEvents() RESULT : tst_BenchlibOneHundredMinimum::threeEvents(): 3.00 events per iteration (total: 192, iterations: 64) PASS : tst_BenchlibOneHundredMinimum::cleanupTestCase() -Totals: 9 passed, 0 failed, 0 skipped +Totals: 3 passed, 0 failed, 0 skipped ********* Finished testing of tst_BenchlibOneHundredMinimum ********* diff --git a/tests/auto/testlib/selftests/selftests.pri b/tests/auto/testlib/selftests/selftests.pri index 2297186f62..0809c4d497 100644 --- a/tests/auto/testlib/selftests/selftests.pri +++ b/tests/auto/testlib/selftests/selftests.pri @@ -3,6 +3,7 @@ SUBPROGRAMS = \ assert \ badxml \ benchlibcallgrind \ + benchlibcounting \ benchlibeventcounter \ benchliboptions \ benchlibtickcounter \ diff --git a/tests/auto/testlib/selftests/selftests.qrc b/tests/auto/testlib/selftests/selftests.qrc index d2f0a50824..baa539a259 100644 --- a/tests/auto/testlib/selftests/selftests.qrc +++ b/tests/auto/testlib/selftests/selftests.qrc @@ -10,6 +10,10 @@ expected_badxml.xml expected_badxml.xunitxml expected_benchlibcallgrind.txt + expected_benchlibcounting.lightxml + expected_benchlibcounting.txt + expected_benchlibcounting.xml + expected_benchlibcounting.xunitxml expected_benchlibeventcounter.lightxml expected_benchlibeventcounter.txt expected_benchlibeventcounter.xml diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 7e671a50f2..decaa55386 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -323,6 +323,7 @@ void tst_Selftests::runSubTest_data() // Only run on platforms where callgrind is available. << "benchlibcallgrind" #endif + << "benchlibcounting" << "benchlibeventcounter" << "benchliboptions" << "cmptest" @@ -399,6 +400,9 @@ void tst_Selftests::runSubTest_data() else if (subtest == "badxml") { arguments << "-eventcounter"; } + else if (subtest == "benchlibcounting") { + arguments << "-eventcounter"; + } else if (subtest == "printdatatags") { arguments << "-datatags"; }