From 123eb803ef1a7af36eefdf9a668f1f29d6597f1d Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Thu, 1 Mar 2012 11:01:49 +1000 Subject: [PATCH] testlib: Add selftests for -v1 and -v2 command-line switches. These tests don't have their own source code but rather reuse the counting selftest with additional command-line options. Note that currently the -v1 switch only changes the plain text output, and the expected xml output is identical to that of the counting test. This may change in the future however. This commit also restores a couple of lists to alphabetical order, where the findtestdata selftest was not sorted into the list correctly. Change-Id: Ie38e255f8029157b34162b3864b5fa66e137d74a Reviewed-by: Rohan McGovern --- .../selftests/expected_verbose1.lightxml | 150 +++++++++++++ .../testlib/selftests/expected_verbose1.txt | 69 ++++++ .../testlib/selftests/expected_verbose1.xml | 153 +++++++++++++ .../selftests/expected_verbose1.xunitxml | 64 ++++++ .../selftests/expected_verbose2.lightxml | 198 +++++++++++++++++ .../testlib/selftests/expected_verbose2.txt | 93 ++++++++ .../testlib/selftests/expected_verbose2.xml | 201 ++++++++++++++++++ .../selftests/expected_verbose2.xunitxml | 89 ++++++++ tests/auto/testlib/selftests/selftests.pri | 2 + tests/auto/testlib/selftests/selftests.qrc | 16 +- .../auto/testlib/selftests/tst_selftests.cpp | 10 +- .../testlib/selftests/verbose1/verbose1.pro | 10 + .../testlib/selftests/verbose2/verbose2.pro | 10 + 13 files changed, 1060 insertions(+), 5 deletions(-) create mode 100644 tests/auto/testlib/selftests/expected_verbose1.lightxml create mode 100644 tests/auto/testlib/selftests/expected_verbose1.txt create mode 100644 tests/auto/testlib/selftests/expected_verbose1.xml create mode 100644 tests/auto/testlib/selftests/expected_verbose1.xunitxml create mode 100644 tests/auto/testlib/selftests/expected_verbose2.lightxml create mode 100644 tests/auto/testlib/selftests/expected_verbose2.txt create mode 100644 tests/auto/testlib/selftests/expected_verbose2.xml create mode 100644 tests/auto/testlib/selftests/expected_verbose2.xunitxml create mode 100644 tests/auto/testlib/selftests/verbose1/verbose1.pro create mode 100644 tests/auto/testlib/selftests/verbose2/verbose2.pro diff --git a/tests/auto/testlib/selftests/expected_verbose1.lightxml b/tests/auto/testlib/selftests/expected_verbose1.lightxml new file mode 100644 index 0000000000..e7b1136417 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose1.lightxml @@ -0,0 +1,150 @@ + + @INSERT_QT_VERSION_HERE@ + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verbose1.txt b/tests/auto/testlib/selftests/expected_verbose1.txt new file mode 100644 index 0000000000..4a0fec51ae --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose1.txt @@ -0,0 +1,69 @@ +********* Start testing of tst_Counting ********* +Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ +INFO : tst_Counting::initTestCase() entering +PASS : tst_Counting::initTestCase() +INFO : tst_Counting::testPassPass() entering +PASS : tst_Counting::testPassPass(row 1) +PASS : tst_Counting::testPassPass(row 2) +INFO : tst_Counting::testPassSkip() entering +PASS : tst_Counting::testPassSkip(row 1) +SKIP : tst_Counting::testPassSkip(row 2) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testPassFail() entering +PASS : tst_Counting::testPassFail(row 1) +FAIL! : tst_Counting::testPassFail(row 2) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testSkipPass() entering +SKIP : tst_Counting::testSkipPass(row 1) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +PASS : tst_Counting::testSkipPass(row 2) +INFO : tst_Counting::testSkipSkip() entering +SKIP : tst_Counting::testSkipSkip(row 1) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +SKIP : tst_Counting::testSkipSkip(row 2) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testSkipFail() entering +SKIP : tst_Counting::testSkipFail(row 1) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +FAIL! : tst_Counting::testSkipFail(row 2) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testFailPass() entering +FAIL! : tst_Counting::testFailPass(row 1) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +PASS : tst_Counting::testFailPass(row 2) +INFO : tst_Counting::testFailSkip() entering +FAIL! : tst_Counting::testFailSkip(row 1) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +SKIP : tst_Counting::testFailSkip(row 2) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testFailFail() entering +FAIL! : tst_Counting::testFailFail(row 1) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testFailFail(row 2) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testFailInInit() entering +PASS : tst_Counting::testFailInInit(before) +FAIL! : tst_Counting::testFailInInit(fail) Fail in init() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(233)] +PASS : tst_Counting::testFailInInit(after) +INFO : tst_Counting::testFailInCleanup() entering +PASS : tst_Counting::testFailInCleanup(before) +QDEBUG : tst_Counting::testFailInCleanup(fail) This test function should execute and then QFAIL in cleanup() +FAIL! : tst_Counting::testFailInCleanup(fail) Fail in cleanup() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(241)] +PASS : tst_Counting::testFailInCleanup(after) +INFO : tst_Counting::testSkipInInit() entering +PASS : tst_Counting::testSkipInInit(before) +SKIP : tst_Counting::testSkipInInit(skip) Skip in init() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(235)] +PASS : tst_Counting::testSkipInInit(after) +INFO : tst_Counting::testSkipInCleanup() entering +PASS : tst_Counting::testSkipInCleanup(before) +QDEBUG : tst_Counting::testSkipInCleanup(skip) This test function should execute and then QSKIP in cleanup() +SKIP : tst_Counting::testSkipInCleanup(skip) Skip in cleanup() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(243)] +PASS : tst_Counting::testSkipInCleanup(after) +INFO : tst_Counting::cleanupTestCase() entering +PASS : tst_Counting::cleanupTestCase() +Totals: 16 passed, 8 failed, 8 skipped +********* Finished testing of tst_Counting ********* diff --git a/tests/auto/testlib/selftests/expected_verbose1.xml b/tests/auto/testlib/selftests/expected_verbose1.xml new file mode 100644 index 0000000000..a97296807d --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose1.xml @@ -0,0 +1,153 @@ + + + + @INSERT_QT_VERSION_HERE@ + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verbose1.xunitxml b/tests/auto/testlib/selftests/expected_verbose1.xunitxml new file mode 100644 index 0000000000..f317ed5923 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose1.xunitxml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verbose2.lightxml b/tests/auto/testlib/selftests/expected_verbose2.lightxml new file mode 100644 index 0000000000..b2d9607ade --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose2.lightxml @@ -0,0 +1,198 @@ + + @INSERT_QT_VERSION_HERE@ + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verbose2.txt b/tests/auto/testlib/selftests/expected_verbose2.txt new file mode 100644 index 0000000000..9d36a5e98e --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose2.txt @@ -0,0 +1,93 @@ +********* Start testing of tst_Counting ********* +Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ +INFO : tst_Counting::initTestCase() entering +PASS : tst_Counting::initTestCase() +INFO : tst_Counting::testPassPass() entering +INFO : tst_Counting::testPassPass(row 1) QVERIFY(true) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(111)] +PASS : tst_Counting::testPassPass(row 1) +INFO : tst_Counting::testPassPass(row 2) QVERIFY(true) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(111)] +PASS : tst_Counting::testPassPass(row 2) +INFO : tst_Counting::testPassSkip() entering +INFO : tst_Counting::testPassSkip(row 1) QVERIFY(true) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(111)] +PASS : tst_Counting::testPassSkip(row 1) +SKIP : tst_Counting::testPassSkip(row 2) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testPassFail() entering +INFO : tst_Counting::testPassFail(row 1) QVERIFY(true) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(111)] +PASS : tst_Counting::testPassFail(row 1) +INFO : tst_Counting::testPassFail(row 2) QVERIFY(false) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testPassFail(row 2) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testSkipPass() entering +SKIP : tst_Counting::testSkipPass(row 1) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testSkipPass(row 2) QVERIFY(true) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(111)] +PASS : tst_Counting::testSkipPass(row 2) +INFO : tst_Counting::testSkipSkip() entering +SKIP : tst_Counting::testSkipSkip(row 1) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +SKIP : tst_Counting::testSkipSkip(row 2) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testSkipFail() entering +SKIP : tst_Counting::testSkipFail(row 1) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testSkipFail(row 2) QVERIFY(false) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testSkipFail(row 2) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testFailPass() entering +INFO : tst_Counting::testFailPass(row 1) QVERIFY(false) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testFailPass(row 1) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testFailPass(row 2) QVERIFY(true) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(111)] +PASS : tst_Counting::testFailPass(row 2) +INFO : tst_Counting::testFailSkip() entering +INFO : tst_Counting::testFailSkip(row 1) QVERIFY(false) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testFailSkip(row 1) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +SKIP : tst_Counting::testFailSkip(row 2) Skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] +INFO : tst_Counting::testFailFail() entering +INFO : tst_Counting::testFailFail(row 1) QVERIFY(false) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testFailFail(row 1) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testFailFail(row 2) QVERIFY(false) + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +FAIL! : tst_Counting::testFailFail(row 2) 'false' returned FALSE. () + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +INFO : tst_Counting::testFailInInit() entering +PASS : tst_Counting::testFailInInit(before) +FAIL! : tst_Counting::testFailInInit(fail) Fail in init() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(233)] +PASS : tst_Counting::testFailInInit(after) +INFO : tst_Counting::testFailInCleanup() entering +PASS : tst_Counting::testFailInCleanup(before) +QDEBUG : tst_Counting::testFailInCleanup(fail) This test function should execute and then QFAIL in cleanup() +FAIL! : tst_Counting::testFailInCleanup(fail) Fail in cleanup() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(241)] +PASS : tst_Counting::testFailInCleanup(after) +INFO : tst_Counting::testSkipInInit() entering +PASS : tst_Counting::testSkipInInit(before) +SKIP : tst_Counting::testSkipInInit(skip) Skip in init() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(235)] +PASS : tst_Counting::testSkipInInit(after) +INFO : tst_Counting::testSkipInCleanup() entering +PASS : tst_Counting::testSkipInCleanup(before) +QDEBUG : tst_Counting::testSkipInCleanup(skip) This test function should execute and then QSKIP in cleanup() +SKIP : tst_Counting::testSkipInCleanup(skip) Skip in cleanup() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(243)] +PASS : tst_Counting::testSkipInCleanup(after) +INFO : tst_Counting::cleanupTestCase() entering +PASS : tst_Counting::cleanupTestCase() +Totals: 16 passed, 8 failed, 8 skipped +********* Finished testing of tst_Counting ********* diff --git a/tests/auto/testlib/selftests/expected_verbose2.xml b/tests/auto/testlib/selftests/expected_verbose2.xml new file mode 100644 index 0000000000..90582e6116 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose2.xml @@ -0,0 +1,201 @@ + + + + @INSERT_QT_VERSION_HERE@ + @INSERT_QT_VERSION_HERE@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verbose2.xunitxml b/tests/auto/testlib/selftests/expected_verbose2.xunitxml new file mode 100644 index 0000000000..558550f17b --- /dev/null +++ b/tests/auto/testlib/selftests/expected_verbose2.xunitxml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/selftests.pri b/tests/auto/testlib/selftests/selftests.pri index 1fc66e6364..c9474419eb 100644 --- a/tests/auto/testlib/selftests/selftests.pri +++ b/tests/auto/testlib/selftests/selftests.pri @@ -39,5 +39,7 @@ SUBPROGRAMS = \ sleep \ strcmp \ subtest \ + verbose1 \ + verbose2 \ warnings \ xunit diff --git a/tests/auto/testlib/selftests/selftests.qrc b/tests/auto/testlib/selftests/selftests.qrc index e39515e5be..03de05fb89 100644 --- a/tests/auto/testlib/selftests/selftests.qrc +++ b/tests/auto/testlib/selftests/selftests.qrc @@ -75,6 +75,10 @@ expected_fetchbogus.txt expected_fetchbogus.xml expected_fetchbogus.xunitxml + expected_findtestdata.lightxml + expected_findtestdata.txt + expected_findtestdata.xml + expected_findtestdata.xunitxml expected_float.txt expected_globaldata.lightxml expected_globaldata.txt @@ -122,10 +126,14 @@ expected_subtest.txt expected_subtest.xml expected_subtest.xunitxml - expected_findtestdata.lightxml - expected_findtestdata.txt - expected_findtestdata.xml - expected_findtestdata.xunitxml + expected_verbose1.lightxml + expected_verbose1.txt + expected_verbose1.xml + expected_verbose1.xunitxml + expected_verbose2.lightxml + expected_verbose2.txt + expected_verbose2.xml + expected_verbose2.xunitxml expected_warnings.lightxml expected_warnings.txt expected_warnings.xml diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 2b90bdb389..5d216992d7 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -346,6 +346,7 @@ void tst_Selftests::runSubTest_data() // Disable this test on Windows, as the run-time will popup dialogs with warnings << "fetchbogus" #endif + << "findtestdata" << "float" << "globaldata" << "longstring" @@ -363,7 +364,8 @@ void tst_Selftests::runSubTest_data() << "sleep" << "strcmp" << "subtest" - << "findtestdata" + << "verbose1" + << "verbose2" << "warnings" << "xunit" ; @@ -413,6 +415,12 @@ void tst_Selftests::runSubTest_data() else if (subtest == "silent") { arguments << "-silent"; } + else if (subtest == "verbose1") { + arguments << "-v1"; + } + else if (subtest == "verbose2") { + arguments << "-v2"; + } // These tests don't work right unless logging plain text to diff --git a/tests/auto/testlib/selftests/verbose1/verbose1.pro b/tests/auto/testlib/selftests/verbose1/verbose1.pro new file mode 100644 index 0000000000..f00ae69d17 --- /dev/null +++ b/tests/auto/testlib/selftests/verbose1/verbose1.pro @@ -0,0 +1,10 @@ +# This test just reuses the counting selftest to show how the output +# differs when the -v1 command-line switch is used. + +SOURCES += ../counting/tst_counting.cpp +QT = core testlib + +mac:CONFIG -= app_bundle +CONFIG -= debug_and_release_target + +TARGET = verbose1 diff --git a/tests/auto/testlib/selftests/verbose2/verbose2.pro b/tests/auto/testlib/selftests/verbose2/verbose2.pro new file mode 100644 index 0000000000..796cdeb975 --- /dev/null +++ b/tests/auto/testlib/selftests/verbose2/verbose2.pro @@ -0,0 +1,10 @@ +# This test just reuses the counting selftest to show how the output +# differs when the -v2 command-line switch is used. + +SOURCES += ../counting/tst_counting.cpp +QT = core testlib + +mac:CONFIG -= app_bundle +CONFIG -= debug_and_release_target + +TARGET = verbose2