AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

testlib: Move subtest argument handling to individual tests

Browse Source 
The test orchestrator shouldn't have to deal with the individual options
needed for each test.

Change-Id: I78bbf4850cc649e625bd08a7aedf02267ba1314d
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Tor Arne Vestbø 2020-02-06 12:07:04 +01:00
parent dbdb3cbc90
commit 8237e39f5d
16 changed files with 115 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tests/auto/testlib/selftests/badxml/tst_badxml.cpp
View File

@ -194,6 +194,9 @@ int main(int argc, char** argv)
args << argv[i];
}
}
args << "-eventcounter";
/*
We just want testlib to output a benchmark result, we don't actually care about the value,
so just do one iteration to save time.

10
tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
View File

@ -79,6 +79,14 @@ void tst_BenchlibCallgrind::twoHundredMillionInstructions()
#endif
}
QTEST_MAIN(tst_BenchlibCallgrind)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-callgrind");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_BenchlibCallgrind)
}
#include "tst_benchlibcallgrind.moc"

11
tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp
View File

@ -59,5 +59,14 @@ void tst_BenchlibCounting::failingBenchmark()
};
}
QTEST_MAIN(tst_BenchlibCounting)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-eventcounter");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_BenchlibCounting)
}
#include "tst_benchlibcounting.moc"

5
tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp
View File

@ -94,6 +94,11 @@ void tst_BenchlibEventCounter::events_data()
int main(int argc, char** argv)
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-eventcounter");
argc = args.size();
argv = const_cast<char**>(&args[0]);
TestEventDispatcher dispatcher;
QCoreApplication app(argc, argv);
tst_BenchlibEventCounter test;

23
tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp
View File

@ -83,6 +83,9 @@ void tst_BenchlibOptions::threeEvents()
int main(int argc, char** argv)
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-eventcounter");
int ret = 0;
TestEventDispatcher dispatcher;
@ -91,29 +94,27 @@ int main(int argc, char** argv)
/* Run with no special arguments. */
{
tst_BenchlibOptions test;
ret += QTest::qExec(&test, argc, argv);
ret += QTest::qExec(&test, args.size(), const_cast<char**>(&args[0]));
}
/* Run with an exact number of iterations. */
{
QVector<char const*> args;
for (int i = 0; i < argc; ++i) args << argv[i];
args << "-iterations";
args << "15";
auto extraArgs = args;
extraArgs.push_back("-iterations");
extraArgs.push_back("15");
tst_BenchlibFifteenIterations test;
ret += QTest::qExec(&test, args.count(), const_cast<char**>(args.data()));
ret += QTest::qExec(&test, extraArgs.size(), const_cast<char**>(&extraArgs[0]));
}
/*
Run until getting a value of at least 100.
*/
{
QVector<char const*> args;
for (int i = 0; i < argc; ++i) args << argv[i];
args << "-minimumvalue";
args << "100";
auto extraArgs = args;
extraArgs.push_back("-minimumvalue");
extraArgs.push_back("100");
tst_BenchlibOneHundredMinimum test;
ret += QTest::qExec(&test, args.count(), const_cast<char**>(args.data()));
ret += QTest::qExec(&test, extraArgs.size(), const_cast<char**>(&extraArgs[0]));
}
return ret;

10
tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
View File

@ -62,6 +62,14 @@ void tst_BenchlibTickCounter::threeBillionTicks()
#endif
}
QTEST_MAIN(tst_BenchlibTickCounter)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-tickcounter");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_BenchlibTickCounter)
}
#include "tst_benchlibtickcounter.moc"

12
tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp
View File

@ -63,6 +63,16 @@ void tst_DataTable::fiveTablePasses_data() const
QTest::newRow("fiveTablePasses_data5") << true;
}
QTEST_MAIN(tst_DataTable)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("fiveTablePasses");
args.push_back("fiveTablePasses:fiveTablePasses_data1");
args.push_back("-v2");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_DataTable)
}
#include "tst_commandlinedata.moc"

12
tests/auto/testlib/selftests/counting/tst_counting.cpp
View File

@ -287,5 +287,15 @@ void tst_Counting::testSkipInCleanup()
qDebug() << "This test function should execute and then QSKIP in cleanup()";
}
QTEST_MAIN(tst_Counting)
int main(int argc, char *argv[])
{
#ifdef TESTLIB_VERBOSITY_ARG
std::vector<const char*> args(argv, argv + argc);
args.push_back(QT_STRINGIFY(TESTLIB_VERBOSITY_ARG));
argc = args.size();
argv = const_cast<char**>(&args[0]);
#endif
QTEST_MAIN_IMPL(tst_Counting)
}
#include "tst_counting.moc"

21
tests/auto/testlib/selftests/generate_expected_output.py
View File

@ -295,23 +295,7 @@ def testEnv(testname,
return data
def generateTestData(testname, clean,
formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap'),
# Make sure this matches tst_Selftests::runSubTest_data():
extraArgs = {
"commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2",
"benchlibcallgrind": "-callgrind",
"benchlibeventcounter": "-eventcounter",
"benchliboptions": "-eventcounter",
"benchlibtickcounter": "-tickcounter",
"badxml": "-eventcounter",
"benchlibcounting": "-eventcounter",
"printdatatags": "-datatags",
"printdatatagswithglobaltags": "-datatags",
"signaldumper": "-vs",
"silent": "-silent",
"verbose1": "-v1",
"verbose2": "-v2",
}):
formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap')):
"""Run one test and save its cleaned results.
Required arguments are the name of the test directory (the binary
@ -330,9 +314,6 @@ def generateTestData(testname, clean,
print(" running", testname)
for format in formats:
cmd = [path, '-' + format]
if testname in extraArgs:
cmd += extraArgs[testname].split()
data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env,
universal_newlines=True).communicate()[0]
with open('expected_' + testname + '.' + format, 'w') as out:

10
tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp
View File

@ -72,6 +72,14 @@ void tst_PrintDataTags::c() const
{
}
QTEST_MAIN(tst_PrintDataTags)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-datatags");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_PrintDataTags)
}
#include "tst_printdatatags.moc"

10
tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp
View File

@ -88,6 +88,14 @@ void tst_PrintDataTagsWithGlobalTags::c() const
{
}
QTEST_MAIN(tst_PrintDataTagsWithGlobalTags)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-datatags");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_PrintDataTagsWithGlobalTags)
}
#include "tst_printdatatagswithglobaltags.moc"

11
tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp
View File

@ -424,5 +424,14 @@ void tst_Signaldumper::deletingSender()
emit signalSlotOwner->signalWithoutParameters();
}
QTEST_MAIN(tst_Signaldumper)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-vs");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_Signaldumper)
}
#include "tst_signaldumper.moc"

11
tests/auto/testlib/selftests/silent/tst_silent.cpp
View File

@ -102,5 +102,14 @@ void tst_Silent::messages()
qFatal("This is a fatal error message that should still appear in silent test output");
}
QTEST_MAIN(tst_Silent)
int main(int argc, char *argv[])
{
std::vector<const char*> args(argv, argv + argc);
args.push_back("-silent");
argc = args.size();
argv = const_cast<char**>(&args[0]);
QTEST_MAIN_IMPL(tst_Silent)
}
#include "tst_silent.moc"

47
tests/auto/testlib/selftests/tst_selftests.cpp
View File

@ -528,49 +528,6 @@ void tst_Selftests::runSubTest_data()
QStringList loggers = loggerSet.loggers;
foreach (QString const& subtest, tests) {
QStringList arguments = loggerSet.arguments;
// Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py:
if (subtest == "commandlinedata") {
arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
}
else if (subtest == "benchlibcallgrind") {
arguments << "-callgrind";
}
else if (subtest == "benchlibeventcounter") {
arguments << "-eventcounter";
}
else if (subtest == "benchliboptions") {
arguments << "-eventcounter";
}
else if (subtest == "benchlibtickcounter") {
arguments << "-tickcounter";
}
else if (subtest == "badxml") {
arguments << "-eventcounter";
}
else if (subtest == "benchlibcounting") {
arguments << "-eventcounter";
}
else if (subtest == "printdatatags") {
arguments << "-datatags";
}
else if (subtest == "printdatatagswithglobaltags") {
arguments << "-datatags";
}
else if (subtest == "signaldumper") {
arguments << "-vs";
}
else if (subtest == "silent") {
arguments << "-silent";
}
else if (subtest == "verbose1") {
arguments << "-v1";
}
else if (subtest == "verbose2") {
arguments << "-v2";
}
// These tests don't work right unless logging plain text to
// standard output, either because they execute multiple test
// objects or because they internally supply arguments to
@ -634,7 +591,7 @@ void tst_Selftests::runSubTest_data()
QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
<< subtest
<< loggers
<< arguments
<< loggerSet.arguments
<< crashes
;
}
@ -690,7 +647,7 @@ static inline QByteArray msgProcessError(const QString &binary, const QStringLis
void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes)
{
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
if (arguments.contains("-callgrind")) {
if (subdir == "benchlibcallgrind") {
QProcess checkProcess;
QStringList args;
args << QLatin1String("--version");

2
tests/auto/testlib/selftests/verbose1/verbose1.pro
View File

@ -4,6 +4,8 @@
SOURCES += ../counting/tst_counting.cpp
QT = core testlib
DEFINES += TESTLIB_VERBOSITY_ARG="-v1"
mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target

2
tests/auto/testlib/selftests/verbose2/verbose2.pro
View File

@ -4,6 +4,8 @@
SOURCES += ../counting/tst_counting.cpp
QT = core testlib
DEFINES += TESTLIB_VERBOSITY_ARG="-v2"
mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target