1223 lines
47 KiB
C++
1223 lines
47 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2016 The Qt Company Ltd.
|
|
** Copyright (C) 2016 Intel Corporation.
|
|
** Contact: https://www.qt.io/licensing/
|
|
**
|
|
** This file is part of the test suite of the Qt Toolkit.
|
|
**
|
|
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
|
|
** Commercial License Usage
|
|
** Licensees holding valid commercial Qt licenses may use this file in
|
|
** accordance with the commercial license agreement provided with the
|
|
** Software or, alternatively, in accordance with the terms contained in
|
|
** a written agreement between you and The Qt Company. For licensing terms
|
|
** and conditions see https://www.qt.io/terms-conditions. For further
|
|
** information use the contact form at https://www.qt.io/contact-us.
|
|
**
|
|
** GNU General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU
|
|
** General Public License version 3 as published by the Free Software
|
|
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
|
** included in the packaging of this file. Please review the following
|
|
** information to ensure the GNU General Public License requirements will
|
|
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
|
**
|
|
** $QT_END_LICENSE$
|
|
**
|
|
****************************************************************************/
|
|
|
|
#include <QtCore/QCoreApplication>
|
|
|
|
#if QT_CONFIG(temporaryfile) && QT_CONFIG(process)
|
|
# define USE_DIFF
|
|
#endif
|
|
#include <QtCore/QXmlStreamReader>
|
|
#include <QtCore/QFileInfo>
|
|
#include <QtCore/QDir>
|
|
#include <QtCore/QTemporaryDir>
|
|
#ifdef USE_DIFF
|
|
# include <QtCore/QTemporaryFile>
|
|
# include <QtCore/QStandardPaths>
|
|
#endif
|
|
#include <QtTest/QtTest>
|
|
|
|
#include <private/cycle_p.h>
|
|
|
|
#include "emulationdetector.h"
|
|
|
|
struct LoggerSet;
|
|
|
|
class tst_Selftests: public QObject
|
|
{
|
|
Q_OBJECT
|
|
public:
|
|
tst_Selftests();
|
|
|
|
private slots:
|
|
void initTestCase();
|
|
void runSubTest_data();
|
|
void runSubTest();
|
|
void cleanup();
|
|
|
|
private:
|
|
void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes);
|
|
bool compareOutput(const QString &logger, const QString &subdir,
|
|
const QByteArray &rawOutput, const QByteArrayList &actual,
|
|
const QByteArrayList &expected,
|
|
QString *errorMessage) const;
|
|
bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
|
|
const QString &actualLine, const QString &expLine,
|
|
QString *errorMessage) const;
|
|
bool checkXml(const QString &logger, QByteArray rawOutput,
|
|
QString *errorMessage) const;
|
|
|
|
QString logName(const QString &logger) const;
|
|
QList<LoggerSet> allLoggerSets() const;
|
|
|
|
QTemporaryDir tempDir;
|
|
QRegularExpression durationRegExp;
|
|
QRegularExpression teamcityLocRegExp;
|
|
};
|
|
|
|
struct BenchmarkResult
|
|
{
|
|
qint64 total;
|
|
qint64 iterations;
|
|
QString unit;
|
|
|
|
inline QString toString() const
|
|
{ return QString("total:%1, unit:%2, iterations:%3").arg(total).arg(unit).arg(iterations); }
|
|
|
|
static BenchmarkResult parse(QString const&, QString*);
|
|
};
|
|
|
|
static QString msgMismatch(const QString &actual, const QString &expected)
|
|
{
|
|
return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
|
|
+ expected + QLatin1Char('\'');
|
|
}
|
|
|
|
static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
|
|
QString *errorMessage)
|
|
{
|
|
// First make sure the iterations and unit match.
|
|
if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
|
|
// Nope - compare whole string for best failure message
|
|
*errorMessage = msgMismatch(r1.toString(), r2.toString());
|
|
return false;
|
|
}
|
|
|
|
// Now check the value. Some variance is allowed, and how much depends on
|
|
// the measured unit.
|
|
qreal variance = 0.;
|
|
if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
|
|
variance = 0.1;
|
|
else if (r1.unit == QLatin1String("instruction reads"))
|
|
variance = 0.001;
|
|
else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
|
|
variance = 0.001;
|
|
|
|
if (variance == 0.) {
|
|
// No variance allowed - compare whole string
|
|
const QString r1S = r1.toString();
|
|
const QString r2S = r2.toString();
|
|
if (r1S != r2S) {
|
|
*errorMessage = msgMismatch(r1S, r2S);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
|
|
// Whoops, didn't match. Compare the whole string for the most useful failure message.
|
|
*errorMessage = msgMismatch(r1.toString(), r2.toString());
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Split the passed block of text into an array of lines, replacing any
|
|
// filenames and line numbers with generic markers to avoid failing the test
|
|
// due to compiler-specific behaviour.
|
|
static QList<QByteArray> splitLines(QByteArray ba)
|
|
{
|
|
ba.replace('\r', "");
|
|
QList<QByteArray> out = ba.split('\n');
|
|
|
|
// Replace any ` file="..."' or ` line="..."' in XML with a generic location.
|
|
static const char *markers[][2] = {
|
|
{ " file=\"", " file=\"__FILE__\"" },
|
|
{ " line=\"", " line=\"__LINE__\"" }
|
|
};
|
|
static const int markerCount = sizeof markers / sizeof markers[0];
|
|
|
|
for (int i = 0; i < out.size(); ++i) {
|
|
QByteArray& line = out[i];
|
|
for (int j = 0; j < markerCount; ++j) {
|
|
int index = line.indexOf(markers[j][0]);
|
|
if (index == -1) {
|
|
continue;
|
|
}
|
|
const int end = line.indexOf('"', index + int(strlen(markers[j][0])));
|
|
if (end == -1) {
|
|
continue;
|
|
}
|
|
line.replace(index, end-index + 1, markers[j][1]);
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
// Return the log format, e.g. for both "stdout txt" and "txt", return "txt'.
|
|
static inline QString logFormat(const QString &logger)
|
|
{
|
|
return (logger.startsWith("stdout") ? logger.mid(7) : logger);
|
|
}
|
|
|
|
// Return the log file name, or an empty string if the log goes to stdout.
|
|
QString tst_Selftests::logName(const QString &logger) const
|
|
{
|
|
return (logger.startsWith("stdout") ? "" : QString(tempDir.path() + "/test_output." + logger));
|
|
}
|
|
|
|
static QString expectedFileNameFromTest(const QString &subdir, const QString &logger)
|
|
{
|
|
return QStringLiteral("expected_") + subdir + QLatin1Char('.') + logFormat(logger);
|
|
}
|
|
|
|
// Load the expected test output for the nominated test (subdir) and logger
|
|
// as an array of lines. If there is no expected output file, return an
|
|
// empty array.
|
|
static QList<QByteArray> expectedResult(const QString &fileName)
|
|
{
|
|
QFile file(QStringLiteral(":/") + fileName);
|
|
if (!file.open(QIODevice::ReadOnly))
|
|
return QList<QByteArray>();
|
|
return splitLines(file.readAll());
|
|
}
|
|
|
|
// Helpers for running the 'diff' tool in case comparison fails
|
|
#ifdef USE_DIFF
|
|
static inline void writeLines(QIODevice &d, const QByteArrayList &lines)
|
|
{
|
|
for (const QByteArray &l : lines) {
|
|
d.write(l);
|
|
d.write("\n");
|
|
}
|
|
}
|
|
#endif // USE_DIFF
|
|
|
|
static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &actual)
|
|
{
|
|
QByteArray result;
|
|
#ifdef USE_DIFF
|
|
# ifndef Q_OS_WIN
|
|
const QString diff = QStandardPaths::findExecutable("diff");
|
|
# else
|
|
const QString diff = QStandardPaths::findExecutable("diff.exe");
|
|
# endif
|
|
if (diff.isEmpty())
|
|
return result;
|
|
QTemporaryFile expectedFile;
|
|
if (!expectedFile.open())
|
|
return result;
|
|
writeLines(expectedFile, expected);
|
|
expectedFile.close();
|
|
QTemporaryFile actualFile;
|
|
if (!actualFile.open())
|
|
return result;
|
|
writeLines(actualFile, actual);
|
|
actualFile.close();
|
|
QProcess diffProcess;
|
|
diffProcess.start(diff, {QLatin1String("-u"), expectedFile.fileName(), actualFile.fileName()});
|
|
if (!diffProcess.waitForStarted())
|
|
return result;
|
|
if (diffProcess.waitForFinished())
|
|
result = diffProcess.readAllStandardOutput();
|
|
else
|
|
diffProcess.kill();
|
|
#endif // USE_DIFF
|
|
return result;
|
|
}
|
|
|
|
// Each test is run with a set of one or more test output loggers.
|
|
// This struct holds information about one such test.
|
|
struct LoggerSet
|
|
{
|
|
LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments)
|
|
: name(_name), loggers(_loggers), arguments(_arguments)
|
|
{ }
|
|
|
|
QString name;
|
|
QStringList loggers;
|
|
QStringList arguments;
|
|
};
|
|
|
|
// This function returns a list of all sets of loggers to be used for
|
|
// running each subtest.
|
|
QList<LoggerSet> tst_Selftests::allLoggerSets() const
|
|
{
|
|
// Note that in order to test XML output to standard output, the subtests
|
|
// must not send output directly to stdout, bypassing Qt's output mechanisms
|
|
// (e.g. via printf), otherwise the output may not be well-formed XML.
|
|
return QList<LoggerSet>()
|
|
// Test with old-style options for a single logger
|
|
<< LoggerSet("old stdout txt",
|
|
QStringList() << "stdout txt",
|
|
QStringList()
|
|
)
|
|
<< LoggerSet("old txt",
|
|
QStringList() << "txt",
|
|
QStringList() << "-o" << logName("txt")
|
|
)
|
|
<< LoggerSet("old stdout xml",
|
|
QStringList() << "stdout xml",
|
|
QStringList() << "-xml"
|
|
)
|
|
<< LoggerSet("old xml",
|
|
QStringList() << "xml",
|
|
QStringList() << "-xml" << "-o" << logName("xml")
|
|
)
|
|
<< LoggerSet("old stdout xunitxml",
|
|
QStringList() << "stdout xunitxml",
|
|
QStringList() << "-xunitxml"
|
|
)
|
|
<< LoggerSet("old xunitxml",
|
|
QStringList() << "xunitxml",
|
|
QStringList() << "-xunitxml" << "-o" << logName("xunitxml")
|
|
)
|
|
<< LoggerSet("old stdout lightxml",
|
|
QStringList() << "stdout lightxml",
|
|
QStringList() << "-lightxml"
|
|
)
|
|
<< LoggerSet("old lightxml",
|
|
QStringList() << "lightxml",
|
|
QStringList() << "-lightxml" << "-o" << logName("lightxml")
|
|
)
|
|
<< LoggerSet("old stdout csv", // benchmarks only
|
|
QStringList() << "stdout csv",
|
|
QStringList() << "-csv")
|
|
<< LoggerSet("old csv", // benchmarks only
|
|
QStringList() << "csv",
|
|
QStringList() << "-csv" << "-o" << logName("csv"))
|
|
<< LoggerSet("old stdout teamcity",
|
|
QStringList() << "stdout teamcity",
|
|
QStringList() << "-teamcity"
|
|
)
|
|
<< LoggerSet("old teamcity",
|
|
QStringList() << "teamcity",
|
|
QStringList() << "-teamcity" << "-o" << logName("teamcity")
|
|
)
|
|
<< LoggerSet("old stdout tap",
|
|
QStringList() << "stdout tap",
|
|
QStringList() << "-tap"
|
|
)
|
|
<< LoggerSet("old tap",
|
|
QStringList() << "tap",
|
|
QStringList() << "-tap" << "-o" << logName("tap")
|
|
)
|
|
// Test with new-style options for a single logger
|
|
<< LoggerSet("new stdout txt",
|
|
QStringList() << "stdout txt",
|
|
QStringList() << "-o" << "-,txt"
|
|
)
|
|
<< LoggerSet("new txt",
|
|
QStringList() << "txt",
|
|
QStringList() << "-o" << logName("txt")+",txt"
|
|
)
|
|
<< LoggerSet("new stdout xml",
|
|
QStringList() << "stdout xml",
|
|
QStringList() << "-o" << "-,xml"
|
|
)
|
|
<< LoggerSet("new xml",
|
|
QStringList() << "xml",
|
|
QStringList() << "-o" << logName("xml")+",xml"
|
|
)
|
|
<< LoggerSet("new stdout xunitxml",
|
|
QStringList() << "stdout xunitxml",
|
|
QStringList() << "-o" << "-,xunitxml"
|
|
)
|
|
<< LoggerSet("new xunitxml",
|
|
QStringList() << "xunitxml",
|
|
QStringList() << "-o" << logName("xunitxml")+",xunitxml"
|
|
)
|
|
<< LoggerSet("new stdout lightxml",
|
|
QStringList() << "stdout lightxml",
|
|
QStringList() << "-o" << "-,lightxml"
|
|
)
|
|
<< LoggerSet("new lightxml",
|
|
QStringList() << "lightxml",
|
|
QStringList() << "-o" << logName("lightxml")+",lightxml"
|
|
)
|
|
<< LoggerSet("new stdout csv", // benchmarks only
|
|
QStringList() << "stdout csv",
|
|
QStringList() << "-o" << "-,csv")
|
|
<< LoggerSet("new csv", // benchmarks only
|
|
QStringList() << "csv",
|
|
QStringList() << "-o" << logName("csv")+",csv")
|
|
<< LoggerSet("new stdout teamcity",
|
|
QStringList() << "stdout teamcity",
|
|
QStringList() << "-o" << "-,teamcity"
|
|
)
|
|
<< LoggerSet("new teamcity",
|
|
QStringList() << "teamcity",
|
|
QStringList() << "-o" << logName("teamcity")+",teamcity"
|
|
)
|
|
<< LoggerSet("new stdout tap",
|
|
QStringList() << "stdout tap",
|
|
QStringList() << "-o" << "-,tap"
|
|
)
|
|
<< LoggerSet("new tap",
|
|
QStringList() << "tap",
|
|
QStringList() << "-o" << logName("tap")+",tap"
|
|
)
|
|
// Test with two loggers (don't test all 32 combinations, just a sample)
|
|
<< LoggerSet("stdout txt + txt",
|
|
QStringList() << "stdout txt" << "txt",
|
|
QStringList() << "-o" << "-,txt"
|
|
<< "-o" << logName("txt")+",txt"
|
|
)
|
|
<< LoggerSet("xml + stdout txt",
|
|
QStringList() << "xml" << "stdout txt",
|
|
QStringList() << "-o" << logName("xml")+",xml"
|
|
<< "-o" << "-,txt"
|
|
)
|
|
<< LoggerSet("txt + xunitxml",
|
|
QStringList() << "txt" << "xunitxml",
|
|
QStringList() << "-o" << logName("txt")+",txt"
|
|
<< "-o" << logName("xunitxml")+",xunitxml"
|
|
)
|
|
<< LoggerSet("lightxml + stdout xunitxml",
|
|
QStringList() << "lightxml" << "stdout xunitxml",
|
|
QStringList() << "-o" << logName("lightxml")+",lightxml"
|
|
<< "-o" << "-,xunitxml"
|
|
)
|
|
// All loggers at the same time (except csv)
|
|
<< LoggerSet("all loggers",
|
|
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml" << "tap",
|
|
QStringList() << "-o" << logName("txt")+",txt"
|
|
<< "-o" << logName("xml")+",xml"
|
|
<< "-o" << logName("lightxml")+",lightxml"
|
|
<< "-o" << "-,txt"
|
|
<< "-o" << logName("xunitxml")+",xunitxml"
|
|
<< "-o" << logName("teamcity")+",teamcity"
|
|
<< "-o" << logName("tap")+",tap"
|
|
)
|
|
;
|
|
}
|
|
|
|
tst_Selftests::tst_Selftests()
|
|
: tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
|
|
, durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>")
|
|
, teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]")
|
|
{}
|
|
|
|
void tst_Selftests::initTestCase()
|
|
{
|
|
QVERIFY2(tempDir.isValid(), qPrintable(tempDir.errorString()));
|
|
//Detect the location of the sub programs
|
|
QString subProgram = QLatin1String("float/float");
|
|
#if defined(Q_OS_WIN)
|
|
subProgram = QLatin1String("float/float.exe");
|
|
#endif
|
|
QString testdataDir = QFINDTESTDATA(subProgram);
|
|
if (testdataDir.lastIndexOf(subProgram) > 0)
|
|
testdataDir = testdataDir.left(testdataDir.lastIndexOf(subProgram));
|
|
else
|
|
testdataDir = QCoreApplication::applicationDirPath();
|
|
// chdir to our testdata path and execute helper apps relative to that.
|
|
QVERIFY2(QDir::setCurrent(testdataDir), qPrintable("Could not chdir to " + testdataDir));
|
|
}
|
|
|
|
void tst_Selftests::runSubTest_data()
|
|
{
|
|
QTest::addColumn<QString>("subdir");
|
|
QTest::addColumn<QStringList>("loggers");
|
|
QTest::addColumn<QStringList>("arguments");
|
|
QTest::addColumn<bool>("crashes");
|
|
|
|
QStringList tests = QStringList()
|
|
#if !defined(Q_OS_WIN)
|
|
// On windows, assert does nothing in release mode and blocks execution
|
|
// with a popup window in debug mode.
|
|
<< "assert"
|
|
#endif
|
|
<< "badxml"
|
|
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
|
|
// Only run on platforms where callgrind is available.
|
|
<< "benchlibcallgrind"
|
|
#endif
|
|
<< "benchlibcounting"
|
|
<< "benchlibeventcounter"
|
|
<< "benchliboptions"
|
|
<< "blacklisted"
|
|
<< "cmptest"
|
|
<< "commandlinedata"
|
|
<< "counting"
|
|
<< "crashes"
|
|
<< "datatable"
|
|
<< "datetime"
|
|
<< "differentexec"
|
|
#if !defined(QT_NO_EXCEPTIONS) && !defined(Q_CC_INTEL) && !defined(Q_OS_WIN)
|
|
// Disable this test on Windows and for intel compiler, as the run-times
|
|
// will popup dialogs with warnings that uncaught exceptions were thrown
|
|
<< "exceptionthrow"
|
|
#endif
|
|
<< "expectfail"
|
|
<< "failcleanup"
|
|
#ifndef Q_OS_WIN // these assert, by design; so same problem as "assert"
|
|
<< "faildatatype"
|
|
<< "failfetchtype"
|
|
#endif
|
|
<< "failinit"
|
|
<< "failinitdata"
|
|
#ifndef Q_OS_WIN // asserts, by design; so same problem as "assert"
|
|
<< "fetchbogus"
|
|
#endif
|
|
<< "findtestdata"
|
|
<< "float"
|
|
<< "globaldata"
|
|
<< "keyboard"
|
|
<< "longstring"
|
|
<< "maxwarnings"
|
|
<< "multiexec"
|
|
<< "pairdiagnostics"
|
|
<< "printdatatags"
|
|
<< "printdatatagswithglobaltags"
|
|
<< "qexecstringlist"
|
|
<< "signaldumper"
|
|
<< "silent"
|
|
<< "singleskip"
|
|
<< "skip"
|
|
<< "skipcleanup"
|
|
<< "skipinit"
|
|
<< "skipinitdata"
|
|
<< "sleep"
|
|
<< "strcmp"
|
|
<< "subtest"
|
|
<< "testlib"
|
|
<< "tuplediagnostics"
|
|
<< "verbose1"
|
|
<< "verbose2"
|
|
#ifndef QT_NO_EXCEPTIONS
|
|
// this test will test nothing if the exceptions are disabled
|
|
<< "verifyexceptionthrown"
|
|
#endif //!QT_NO_EXCEPTIONS
|
|
<< "warnings"
|
|
<< "watchdog"
|
|
<< "xunit"
|
|
;
|
|
|
|
// These tests are affected by timing and whether the CPU tick counter
|
|
// is monotonically increasing. They won't work on some machines so
|
|
// leave them off by default. Feel free to enable them for your own
|
|
// testing by setting the QTEST_ENABLE_EXTRA_SELFTESTS environment
|
|
// variable to something non-empty.
|
|
if (!qgetenv("QTEST_ENABLE_EXTRA_SELFTESTS").isEmpty())
|
|
tests << "benchlibtickcounter"
|
|
<< "benchlibwalltime"
|
|
;
|
|
|
|
foreach (LoggerSet const& loggerSet, allLoggerSets()) {
|
|
QStringList loggers = loggerSet.loggers;
|
|
|
|
foreach (QString const& subtest, tests) {
|
|
QStringList arguments = loggerSet.arguments;
|
|
// Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py:
|
|
if (subtest == "commandlinedata") {
|
|
arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
|
|
}
|
|
else if (subtest == "benchlibcallgrind") {
|
|
arguments << "-callgrind";
|
|
}
|
|
else if (subtest == "benchlibeventcounter") {
|
|
arguments << "-eventcounter";
|
|
}
|
|
else if (subtest == "benchliboptions") {
|
|
arguments << "-eventcounter";
|
|
}
|
|
else if (subtest == "benchlibtickcounter") {
|
|
arguments << "-tickcounter";
|
|
}
|
|
else if (subtest == "badxml") {
|
|
arguments << "-eventcounter";
|
|
}
|
|
else if (subtest == "benchlibcounting") {
|
|
arguments << "-eventcounter";
|
|
}
|
|
else if (subtest == "printdatatags") {
|
|
arguments << "-datatags";
|
|
}
|
|
else if (subtest == "printdatatagswithglobaltags") {
|
|
arguments << "-datatags";
|
|
}
|
|
else if (subtest == "signaldumper") {
|
|
arguments << "-vs";
|
|
}
|
|
else if (subtest == "silent") {
|
|
arguments << "-silent";
|
|
}
|
|
else if (subtest == "verbose1") {
|
|
arguments << "-v1";
|
|
}
|
|
else if (subtest == "verbose2") {
|
|
arguments << "-v2";
|
|
}
|
|
|
|
|
|
// These tests don't work right unless logging plain text to
|
|
// standard output, either because they execute multiple test
|
|
// objects or because they internally supply arguments to
|
|
// themselves.
|
|
if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") {
|
|
if (subtest == "differentexec") {
|
|
continue;
|
|
}
|
|
if (subtest == "multiexec") {
|
|
continue;
|
|
}
|
|
if (subtest == "qexecstringlist") {
|
|
continue;
|
|
}
|
|
if (subtest == "benchliboptions") {
|
|
continue;
|
|
}
|
|
if (subtest == "blacklisted") {
|
|
continue;
|
|
}
|
|
if (subtest == "printdatatags") {
|
|
continue;
|
|
}
|
|
if (subtest == "printdatatagswithglobaltags") {
|
|
continue;
|
|
}
|
|
if (subtest == "silent") {
|
|
continue;
|
|
}
|
|
// `crashes' will not output valid XML on platforms without a crash handler
|
|
if (subtest == "crashes") {
|
|
continue;
|
|
}
|
|
// this test prints out some floats in the testlog and the formatting is
|
|
// platform-specific and hard to predict.
|
|
if (subtest == "float") {
|
|
continue;
|
|
}
|
|
// these tests are quite slow, and running them for all the loggers significantly
|
|
// increases the overall test time. They do not really relate to logging, so it
|
|
// should be safe to run them just for the stdout loggers.
|
|
if (subtest == "benchlibcallgrind" || subtest == "sleep") {
|
|
continue;
|
|
}
|
|
}
|
|
if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt")))
|
|
continue; // XML only, do not mix txt and XML for encoding test.
|
|
|
|
if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib"))
|
|
continue;
|
|
|
|
if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib"))
|
|
continue; // Skip benchmark for TeamCity logger
|
|
|
|
// Keep in sync with generateTestData()'s crashers in generate_expected_output.py:
|
|
const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
|
|
|| subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
|
|
|| subtest == QLatin1String("faildatatype") || subtest == QLatin1String("failfetchtype")
|
|
|| subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
|
|
|| subtest == QLatin1String("blacklisted") || subtest == QLatin1String("watchdog");
|
|
QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
|
|
<< subtest
|
|
<< loggers
|
|
<< arguments
|
|
<< crashes
|
|
;
|
|
}
|
|
}
|
|
}
|
|
|
|
#if QT_CONFIG(process)
|
|
|
|
static QProcessEnvironment processEnvironment()
|
|
{
|
|
static QProcessEnvironment result;
|
|
if (result.isEmpty()) {
|
|
const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
|
|
const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH");
|
|
foreach (const QString &key, systemEnvironment.keys()) {
|
|
const bool useVariable = key == QLatin1String("PATH") || key == QLatin1String("QT_QPA_PLATFORM")
|
|
#if defined(Q_OS_QNX)
|
|
|| key == QLatin1String("GRAPHICS_ROOT") || key == QLatin1String("TZ")
|
|
#elif defined(Q_OS_UNIX)
|
|
|| key == QLatin1String("HOME") || key == QLatin1String("USER") // Required for X11 on openSUSE
|
|
|| key == QLatin1String("QEMU_SET_ENV") || key == QLatin1String("QEMU_LD_PREFIX") // Required for QEMU
|
|
# if !defined(Q_OS_MAC)
|
|
|| key == QLatin1String("DISPLAY") || key == QLatin1String("XAUTHLOCALHOSTNAME")
|
|
|| key.startsWith(QLatin1String("XDG_"))
|
|
# endif // !Q_OS_MAC
|
|
#endif // Q_OS_UNIX
|
|
#ifdef __COVERAGESCANNER__
|
|
|| key == QLatin1String("QT_TESTCOCOON_ACTIVE")
|
|
#endif
|
|
|| ( preserveLibPath && (key == QLatin1String("QT_PLUGIN_PATH")
|
|
|| key == QLatin1String("LD_LIBRARY_PATH")))
|
|
;
|
|
if (useVariable)
|
|
result.insert(key, systemEnvironment.value(key));
|
|
}
|
|
// Avoid interference from any qtlogging.ini files, e.g. in /etc/xdg/QtProject/:
|
|
result.insert(QStringLiteral("QT_LOGGING_RULES"),
|
|
// Must match generate_expected_output.py's main()'s value:
|
|
QStringLiteral("*.debug=true;qt.*=false"));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
static inline QByteArray msgProcessError(const QString &binary, const QStringList &args,
|
|
const QProcessEnvironment &e, const QString &what)
|
|
{
|
|
QString result;
|
|
QTextStream(&result) <<"Error running " << binary << ' ' << args.join(' ')
|
|
<< " with environment " << e.toStringList().join(' ') << ": " << what;
|
|
return result.toLocal8Bit();
|
|
}
|
|
|
|
void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes)
|
|
{
|
|
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
|
|
if (arguments.contains("-callgrind")) {
|
|
QProcess checkProcess;
|
|
QStringList args;
|
|
args << QLatin1String("--version");
|
|
checkProcess.start(QLatin1String("valgrind"), args);
|
|
if (!checkProcess.waitForFinished(-1))
|
|
QSKIP(QString("Valgrind broken or not available. Not running %1 test!").arg(subdir).toLocal8Bit());
|
|
}
|
|
#endif
|
|
|
|
QProcess proc;
|
|
QProcessEnvironment environment = processEnvironment();
|
|
// Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py:
|
|
if (crashes) {
|
|
environment.insert("QTEST_DISABLE_CORE_DUMP", "1");
|
|
environment.insert("QTEST_DISABLE_STACK_DUMP", "1");
|
|
if (subdir == QLatin1String("watchdog"))
|
|
environment.insert("QTEST_FUNCTION_TIMEOUT", "100");
|
|
}
|
|
proc.setProcessEnvironment(environment);
|
|
const QString path = subdir + QLatin1Char('/') + subdir;
|
|
proc.start(path, arguments);
|
|
QVERIFY2(proc.waitForStarted(), msgProcessError(path, arguments, environment, QStringLiteral("Cannot start: ") + proc.errorString()));
|
|
QVERIFY2(proc.waitForFinished(), msgProcessError(path, arguments, environment, QStringLiteral("Timed out: ") + proc.errorString()));
|
|
if (!crashes) {
|
|
QVERIFY2(proc.exitStatus() == QProcess::NormalExit,
|
|
msgProcessError(path, arguments, environment,
|
|
QStringLiteral("Crashed: ") + proc.errorString()
|
|
+ QStringLiteral(": ") + QString::fromLocal8Bit(proc.readAllStandardError())));
|
|
}
|
|
|
|
QList<QByteArray> actualOutputs;
|
|
for (int i = 0; i < loggers.count(); ++i) {
|
|
QString logFile = logName(loggers[i]);
|
|
QByteArray out;
|
|
if (logFile.isEmpty()) {
|
|
out = proc.readAllStandardOutput();
|
|
} else {
|
|
QFile file(logFile);
|
|
if (file.open(QIODevice::ReadOnly))
|
|
out = file.readAll();
|
|
}
|
|
actualOutputs << out;
|
|
}
|
|
|
|
const QByteArray err(proc.readAllStandardError());
|
|
|
|
// Some tests may output unpredictable strings to stderr, which we'll ignore.
|
|
//
|
|
// For instance, uncaught exceptions on Windows might say (depending on Windows
|
|
// version and JIT debugger settings):
|
|
// "This application has requested the Runtime to terminate it in an unusual way.
|
|
// Please contact the application's support team for more information."
|
|
//
|
|
// Also, tests which use valgrind may generate warnings if the toolchain is
|
|
// newer than the valgrind version, such that valgrind can't understand the
|
|
// debug information on the binary.
|
|
if (subdir != QLatin1String("exceptionthrow")
|
|
&& subdir != QLatin1String("cmptest") // QImage comparison requires QGuiApplication
|
|
&& subdir != QLatin1String("fetchbogus")
|
|
&& subdir != QLatin1String("watchdog")
|
|
&& subdir != QLatin1String("xunit")
|
|
#ifdef Q_CC_MINGW
|
|
&& subdir != QLatin1String("blacklisted") // calls qFatal()
|
|
&& subdir != QLatin1String("silent") // calls qFatal()
|
|
#endif
|
|
#ifdef Q_OS_LINUX
|
|
// QEMU outputs to stderr about uncaught signals
|
|
&& !(EmulationDetector::isRunningArmOnX86() &&
|
|
(subdir == QLatin1String("assert")
|
|
|| subdir == QLatin1String("blacklisted")
|
|
|| subdir == QLatin1String("crashes")
|
|
|| subdir == QLatin1String("faildatatype")
|
|
|| subdir == QLatin1String("failfetchtype")
|
|
|| subdir == QLatin1String("silent")
|
|
)
|
|
)
|
|
#endif
|
|
&& subdir != QLatin1String("benchlibcallgrind"))
|
|
QVERIFY2(err.isEmpty(), err.constData());
|
|
|
|
for (int n = 0; n < loggers.count(); ++n) {
|
|
QString logger = loggers[n];
|
|
if (n == 0 && subdir == QLatin1String("crashes")) {
|
|
QByteArray &actual = actualOutputs[0];
|
|
#ifndef Q_OS_WIN
|
|
// Remove digits of times to match the expected file.
|
|
const QLatin1String timePattern("Function time:");
|
|
int timePos = actual.indexOf(timePattern);
|
|
if (timePos >= 0) {
|
|
timePos += timePattern.size();
|
|
const int nextLinePos = actual.indexOf('\n', timePos);
|
|
for (int c = (nextLinePos != -1 ? nextLinePos : actual.size()) - 1; c >= timePos; --c) {
|
|
if (actual.at(c) >= '0' && actual.at(c) <= '9')
|
|
actual.remove(c, 1);
|
|
}
|
|
}
|
|
#else // !Q_OS_WIN
|
|
// Remove stack trace which is output to stdout.
|
|
const int exceptionLogStart = actual.indexOf("A crash occurred in ");
|
|
if (exceptionLogStart >= 0)
|
|
actual.truncate(exceptionLogStart);
|
|
#endif // Q_OS_WIN
|
|
}
|
|
|
|
QList<QByteArray> res = splitLines(actualOutputs[n]);
|
|
QString errorMessage;
|
|
QString expectedFileName = expectedFileNameFromTest(subdir, logger);
|
|
QByteArrayList exp = expectedResult(expectedFileName);
|
|
if (!exp.isEmpty()) {
|
|
if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) {
|
|
errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ")
|
|
+ expectedFileName + QLatin1Char(' '));
|
|
errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]);
|
|
const QByteArray diff = runDiff(exp, res);
|
|
if (!diff.isEmpty())
|
|
errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff);
|
|
QFAIL(qPrintable(errorMessage));
|
|
}
|
|
} else {
|
|
// For the "crashes" and other tests, there are multiple versions of the
|
|
// expected output. Loop until a matching one is found.
|
|
bool ok = false;
|
|
for (int i = 1; !ok; ++i) {
|
|
expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger);
|
|
const QByteArrayList exp = expectedResult(expectedFileName);
|
|
if (exp.isEmpty())
|
|
break;
|
|
QString errorMessage2;
|
|
ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2);
|
|
if (!ok)
|
|
errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2;
|
|
}
|
|
if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output.
|
|
errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir);
|
|
errorMessage += QLatin1String("\nActual:\n");
|
|
QDebug(&errorMessage) << actualOutputs[n];
|
|
QFAIL(qPrintable(errorMessage));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
|
|
static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }
|
|
|
|
bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
|
|
const QByteArray &rawOutput, const QByteArrayList &actual,
|
|
const QByteArrayList &expected,
|
|
QString *errorMessage) const
|
|
{
|
|
|
|
if (actual.size() != expected.size()) {
|
|
*errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.")
|
|
.arg(actual.size()).arg(expected.size());
|
|
return false;
|
|
}
|
|
|
|
// For xml output formats, verify that the log is valid XML.
|
|
if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
|
|
return false;
|
|
|
|
// Verify that the actual output is an acceptable match for the
|
|
// expected output.
|
|
|
|
const QString qtVersion = QLatin1String(QT_VERSION_STR);
|
|
bool benchmark = false;
|
|
for (int i = 0, size = actual.size(); i < size; ++i) {
|
|
const QByteArray &actualLineBA = actual.at(i);
|
|
// the __FILE__ __LINE__ output is compiler dependent, skip it
|
|
if (actualLineBA.startsWith(" Loc: [") && actualLineBA.endsWith(")]"))
|
|
continue;
|
|
if (actualLineBA.endsWith(" : failure location"))
|
|
continue;
|
|
|
|
if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string
|
|
|| actualLineBA.startsWith(" <QtBuild") // XML, Light XML build string
|
|
|| (actualLineBA.startsWith(" <property value=") && actualLineBA.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
|
|
continue;
|
|
}
|
|
|
|
QString actualLine = QString::fromLatin1(actualLineBA);
|
|
QString expectedLine = QString::fromLatin1(expected.at(i));
|
|
expectedLine.replace(qtVersionPlaceHolder(), qtVersion);
|
|
|
|
// Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
|
|
if (logger.endsWith(QLatin1String("teamcity"))) {
|
|
actualLine.replace(teamcityLocRegExp, teamCityLocation());
|
|
expectedLine.replace(teamcityLocRegExp, teamCityLocation());
|
|
}
|
|
|
|
if (logger.endsWith(QLatin1String("tap"))) {
|
|
if (expectedLine.contains(QLatin1String("at:"))
|
|
|| expectedLine.contains(QLatin1String("file:"))
|
|
|| expectedLine.contains(QLatin1String("line:")))
|
|
actualLine = expectedLine;
|
|
}
|
|
|
|
if (!compareLine(logger, subdir, benchmark, actualLine,
|
|
expectedLine, errorMessage)) {
|
|
errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)
|
|
+ QLatin1String(": "));
|
|
return false;
|
|
}
|
|
|
|
benchmark = actualLineBA.startsWith("RESULT : ");
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
|
|
bool benchmark,
|
|
const QString &actualLine, const QString &expectedLine,
|
|
QString *errorMessage) const
|
|
{
|
|
if (actualLine == expectedLine)
|
|
return true;
|
|
|
|
if ((subdir == QLatin1String("assert")
|
|
|| subdir == QLatin1String("faildatatype") || subdir == QLatin1String("failfetchtype"))
|
|
&& actualLine.contains(QLatin1String("ASSERT: "))
|
|
&& expectedLine.contains(QLatin1String("ASSERT: "))) {
|
|
// Q_ASSERT uses __FILE__, the exact contents of which are
|
|
// undefined. If have we something that looks like a Q_ASSERT and we
|
|
// were expecting to see a Q_ASSERT, we'll skip the line.
|
|
return true;
|
|
}
|
|
|
|
if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce"))) {
|
|
// On some platforms we compile without RTTI, and as a result we never throw an exception
|
|
if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
|
|
*errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult"))
|
|
|| (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) {
|
|
// Don't do a literal comparison for benchmark results, since
|
|
// results have some natural variance.
|
|
QString error;
|
|
BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
|
|
if (!error.isEmpty()) {
|
|
*errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine);
|
|
return false;
|
|
}
|
|
BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
|
|
if (!error.isEmpty()) {
|
|
*errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine);
|
|
return false;
|
|
}
|
|
return compareBenchmarkResult(actualResult, expectedResult, errorMessage);
|
|
}
|
|
|
|
if (actualLine.startsWith(QLatin1String(" <Duration msecs="))
|
|
|| actualLine.startsWith(QLatin1String("<Duration msecs="))) {
|
|
QRegularExpressionMatch match = durationRegExp.match(actualLine);
|
|
if (match.hasMatch())
|
|
return true;
|
|
*errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine);
|
|
return false;
|
|
}
|
|
|
|
if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:")))
|
|
return true;
|
|
|
|
const QLatin1String pointerPlaceholder("_POINTER_");
|
|
if (expectedLine.contains(pointerPlaceholder)
|
|
&& (expectedLine.contains(QLatin1String("Signal: "))
|
|
|| expectedLine.contains(QLatin1String("Slot: ")))) {
|
|
QString actual = actualLine;
|
|
// We don't care about the pointer of the object to whom the signal belongs, so we
|
|
// replace it with _POINTER_, e.g.:
|
|
// Signal: SignalSlotClass(7ffd72245410) signalWithoutParameters ()
|
|
// Signal: QThread(7ffd72245410) started ()
|
|
// After this instance pointer we may have further pointers and
|
|
// references (with an @ prefix) as parameters of the signal or
|
|
// slot being invoked.
|
|
// Signal: SignalSlotClass(_POINTER_) qStringRefSignal ((QString&)@55f5fbb8dd40)
|
|
actual.replace(QRegularExpression("\\b[a-f0-9]{8,}\\b"), pointerPlaceholder);
|
|
// Also change QEventDispatcher{Glib,Win32,etc.} to QEventDispatcherPlatform
|
|
actual.replace(QRegularExpression("\\b(QEventDispatcher)\\w+\\b"), QLatin1String("\\1Platform"));
|
|
if (actual != expectedLine) {
|
|
*errorMessage = msgMismatch(actual, expectedLine);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
*errorMessage = msgMismatch(actualLine, expectedLine);
|
|
return false;
|
|
}
|
|
|
|
bool tst_Selftests::checkXml(const QString &logger, QByteArray xml,
|
|
QString *errorMessage) const
|
|
{
|
|
// lightxml intentionally skips the root element, which technically makes it
|
|
// not valid XML.
|
|
// We'll add that ourselves for the purpose of validation.
|
|
if (logger.endsWith(QLatin1String("lightxml"))) {
|
|
xml.prepend("<root>");
|
|
xml.append("</root>");
|
|
}
|
|
|
|
QXmlStreamReader reader(xml);
|
|
while (!reader.atEnd())
|
|
reader.readNext();
|
|
|
|
if (reader.hasError()) {
|
|
const int lineNumber = int(reader.lineNumber());
|
|
const QByteArray line = xml.split('\n').value(lineNumber - 1);
|
|
*errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4")
|
|
.arg(lineNumber).arg(reader.columnNumber())
|
|
.arg(QString::fromLatin1(line), reader.errorString());
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
#endif // QT_CONFIG(process)
|
|
|
|
void tst_Selftests::runSubTest()
|
|
{
|
|
#if !QT_CONFIG(process)
|
|
QSKIP("This test requires QProcess support");
|
|
#else
|
|
QFETCH(QString, subdir);
|
|
QFETCH(QStringList, loggers);
|
|
QFETCH(QStringList, arguments);
|
|
QFETCH(bool, crashes);
|
|
|
|
doRunSubTest(subdir, loggers, arguments, crashes);
|
|
#endif // QT_CONFIG(process)
|
|
}
|
|
|
|
// attribute must contain ="
|
|
QString extractXmlAttribute(const QString &line, const char *attribute)
|
|
{
|
|
int index = line.indexOf(attribute);
|
|
if (index == -1)
|
|
return QString();
|
|
const int attributeLength = int(strlen(attribute));
|
|
const int end = line.indexOf('"', index + attributeLength);
|
|
if (end == -1)
|
|
return QString();
|
|
|
|
const QString result = line.mid(index + attributeLength, end - index - attributeLength);
|
|
if (result.isEmpty())
|
|
return ""; // ensure empty but not null
|
|
return result;
|
|
}
|
|
|
|
// Parse line into the BenchmarkResult it represents.
|
|
BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
|
|
{
|
|
if (error) *error = QString();
|
|
BenchmarkResult out;
|
|
|
|
QString remaining = line.trimmed();
|
|
|
|
if (remaining.isEmpty()) {
|
|
if (error) *error = "Line is empty";
|
|
return out;
|
|
}
|
|
|
|
if (line.startsWith("<BenchmarkResult ")) {
|
|
// XML result
|
|
// format:
|
|
// <BenchmarkResult metric="$unit" tag="$tag" value="$total" iterations="$iterations" />
|
|
if (!line.endsWith("/>")) {
|
|
if (error) *error = "unterminated XML";
|
|
return out;
|
|
}
|
|
|
|
QString unit = extractXmlAttribute(line, " metric=\"");
|
|
QString sTotal = extractXmlAttribute(line, " value=\"");
|
|
QString sIterations = extractXmlAttribute(line, " iterations=\"");
|
|
if (unit.isNull() || sTotal.isNull() || sIterations.isNull()) {
|
|
if (error) *error = "XML snippet did not contain all required values";
|
|
return out;
|
|
}
|
|
|
|
bool ok;
|
|
double total = sTotal.toDouble(&ok);
|
|
if (!ok) {
|
|
if (error) *error = sTotal + " is not a valid number";
|
|
return out;
|
|
}
|
|
double iterations = sIterations.toDouble(&ok);
|
|
if (!ok) {
|
|
if (error) *error = sIterations + " is not a valid number";
|
|
return out;
|
|
}
|
|
|
|
out.unit = unit;
|
|
out.total = total;
|
|
out.iterations = iterations;
|
|
return out;
|
|
}
|
|
|
|
if (line.startsWith('"')) {
|
|
// CSV result
|
|
// format:
|
|
// "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
|
|
QStringList split = line.split(',');
|
|
if (split.count() != 6) {
|
|
if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
|
|
return out;
|
|
}
|
|
|
|
bool ok;
|
|
double total = split.at(4).toDouble(&ok);
|
|
if (!ok) {
|
|
if (error) *error = split.at(4) + " is not a valid number";
|
|
return out;
|
|
}
|
|
double iterations = split.at(5).toDouble(&ok);
|
|
if (!ok) {
|
|
if (error) *error = split.at(5) + " is not a valid number";
|
|
return out;
|
|
}
|
|
|
|
out.unit = split.at(2);
|
|
out.total = total;
|
|
out.iterations = iterations;
|
|
return out;
|
|
}
|
|
|
|
// Text result
|
|
// This code avoids using a QRegExp because QRegExp might be broken.
|
|
// Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)
|
|
|
|
QString sFirstNumber;
|
|
while (!remaining.isEmpty() && !remaining.at(0).isSpace()) {
|
|
sFirstNumber += remaining.at(0);
|
|
remaining.remove(0,1);
|
|
}
|
|
remaining = remaining.trimmed();
|
|
|
|
// 4,000 -> 4000
|
|
sFirstNumber.remove(',');
|
|
|
|
// Should now be parseable as floating point
|
|
bool ok;
|
|
double firstNumber = sFirstNumber.toDouble(&ok);
|
|
if (!ok) {
|
|
if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";
|
|
return out;
|
|
}
|
|
|
|
// Remaining: msec per iteration (total: 4000, iterations: 1)
|
|
static const char periterbit[] = " per iteration (total: ";
|
|
QString unit;
|
|
while (!remaining.startsWith(periterbit) && !remaining.isEmpty()) {
|
|
unit += remaining.at(0);
|
|
remaining.remove(0,1);
|
|
}
|
|
if (remaining.isEmpty()) {
|
|
if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";
|
|
return out;
|
|
}
|
|
|
|
remaining = remaining.mid(sizeof(periterbit)-1);
|
|
|
|
// Remaining: 4,000, iterations: 1)
|
|
static const char itersbit[] = ", iterations: ";
|
|
QString sTotal;
|
|
while (!remaining.startsWith(itersbit) && !remaining.isEmpty()) {
|
|
sTotal += remaining.at(0);
|
|
remaining.remove(0,1);
|
|
}
|
|
if (remaining.isEmpty()) {
|
|
if (error) *error = "Could not find pattern: '<number>, iterations: '";
|
|
return out;
|
|
}
|
|
|
|
remaining = remaining.mid(sizeof(itersbit)-1);
|
|
|
|
// 4,000 -> 4000
|
|
sTotal.remove(',');
|
|
|
|
double total = sTotal.toDouble(&ok);
|
|
if (!ok) {
|
|
if (error) *error = sTotal + " (total) is not a valid number";
|
|
return out;
|
|
}
|
|
|
|
// Remaining: 1)
|
|
QString sIters;
|
|
while (remaining != QLatin1String(")") && !remaining.isEmpty()) {
|
|
sIters += remaining.at(0);
|
|
remaining.remove(0,1);
|
|
}
|
|
if (remaining.isEmpty()) {
|
|
if (error) *error = "Could not find pattern: '<num>)'";
|
|
return out;
|
|
}
|
|
qint64 iters = sIters.toLongLong(&ok);
|
|
if (!ok) {
|
|
if (error) *error = sIters + " (iterations) is not a valid integer";
|
|
return out;
|
|
}
|
|
|
|
double calcFirstNumber = double(total)/double(iters);
|
|
if (!qFuzzyCompare(firstNumber, calcFirstNumber)) {
|
|
if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(calcFirstNumber).arg(firstNumber);
|
|
return out;
|
|
}
|
|
|
|
out.total = total;
|
|
out.unit = unit;
|
|
out.iterations = iters;
|
|
return out;
|
|
}
|
|
|
|
void tst_Selftests::cleanup()
|
|
{
|
|
QFETCH(QStringList, loggers);
|
|
|
|
// Remove the test output files
|
|
for (int i = 0; i < loggers.count(); ++i) {
|
|
QString logFileName = logName(loggers[i]);
|
|
if (!logFileName.isEmpty()) {
|
|
QFile logFile(logFileName);
|
|
if (logFile.exists())
|
|
QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: ").arg(logFileName, logFile.errorString())));
|
|
}
|
|
}
|
|
}
|
|
|
|
QTEST_MAIN(tst_Selftests)
|
|
|
|
#include "tst_selftests.moc"
|