AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
5f384bd39c
qt5base-lts/tests/auto/testlib/selftests/tst_selftests.cpp
Edward Welbourne 150c6fb74b Add testlib selftests for double and for non-finite float and double 
Tidied up the existing float tests in the process.
(In particular, s/SUCCESS/PASS/ since that matches real test output.)
These verify that QCOMPARE() handles floats and doubles as intended.
Extended the existing qFuzzyCompare tests to probe the boundaries of
the ranges of values of both types, in the process.

Revised the toString<double> that qCompare() uses to give enough
precision to actually show some of the differences being tested there
(12 digits, to match what qFuzzyCompare tests, so as to show different
values rather than, e.g. 1e12 for both expected and actual) and to
give consistent results for infinities and NaN (MinGW had eccentric
versions for these, leading to different output from tests, which thus
failed); did the latter also for toString<float> and fixed stray zeros
in MinGW's exponents (which made a kludge in tst_selftest.cpp
redundant, so I removed that, too).

That's further complicated handling of floating-point types, so let's
just keep an eye on how expensive that's getting by adding a benchmark
test for QTest::toString().  Unfortunately, default settings only get
runs that take modest numbers of milliseconds (some as low as 40)
while increasing this with -minumumvalue 100 or more gets the process
killed - and I'm unable to find out who's doing the killing (it's not
QProcess::kill, ::kill or the QtTest WatchDog, as far as I can tell).
So results are rather noisy; the integral tests exhibit speed-ups by
factors up to 5, and slow-downs by factors up to 100, between runs
with and without this change, which does not affec the integral tests.
The relatively modest slow-downs and speed-ups in the floating point
tests thus seem likely to be happenstance rather than signal.

Change-Id: I4a6bbbab6a43bf14a4089e96238a7c8da2c3127e
Reviewed-by: Ulf Hermann <ulf.hermann@qt.io>
2019-02-06 10:11:22 +00:00

1185 lines
45 KiB
C++
Raw Blame History

/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2016 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the test suite of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include <QtCore/QCoreApplication>
#if QT_CONFIG(temporaryfile) && QT_CONFIG(process)
# define USE_DIFF
#endif
#include <QtCore/QXmlStreamReader>
#include <QtCore/QFileInfo>
#include <QtCore/QDir>
#include <QtCore/QTemporaryDir>
#ifdef USE_DIFF
# include <QtCore/QTemporaryFile>
# include <QtCore/QStandardPaths>
#endif
#include <QtTest/QtTest>
#include <private/cycle_p.h>
#include "emulationdetector.h"
struct LoggerSet;
class tst_Selftests: public QObject
{
Q_OBJECT
public:
tst_Selftests();
private slots:
void initTestCase();
void runSubTest_data();
void runSubTest();
void cleanup();
private:
void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes);
bool compareOutput(const QString &logger, const QString &subdir,
const QByteArray &rawOutput, const QByteArrayList &actual,
const QByteArrayList &expected,
QString *errorMessage) const;
bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
const QString &actualLine, const QString &expLine,
QString *errorMessage) const;
bool checkXml(const QString &logger, QByteArray rawOutput,
QString *errorMessage) const;
QString logName(const QString &logger) const;
QList<LoggerSet> allLoggerSets() const;
QTemporaryDir tempDir;
QRegularExpression durationRegExp;
QRegularExpression teamcityLocRegExp;
};
struct BenchmarkResult
{
qint64 total;
qint64 iterations;
QString unit;
inline QString toString() const
{ return QString("total:%1, unit:%2, iterations:%3").arg(total).arg(unit).arg(iterations); }
static BenchmarkResult parse(QString const&, QString*);
};
static QString msgMismatch(const QString &actual, const QString &expected)
{
return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
+ expected + QLatin1Char('\'');
}
static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
QString *errorMessage)
{
// First make sure the iterations and unit match.
if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
// Nope - compare whole string for best failure message
*errorMessage = msgMismatch(r1.toString(), r2.toString());
return false;
}
// Now check the value. Some variance is allowed, and how much depends on
// the measured unit.
qreal variance = 0.;
if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
variance = 0.1;
else if (r1.unit == QLatin1String("instruction reads"))
variance = 0.001;
else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
variance = 0.001;
if (variance == 0.) {
// No variance allowed - compare whole string
const QString r1S = r1.toString();
const QString r2S = r2.toString();
if (r1S != r2S) {
*errorMessage = msgMismatch(r1S, r2S);
return false;
}
return true;
}
if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
// Whoops, didn't match. Compare the whole string for the most useful failure message.
*errorMessage = msgMismatch(r1.toString(), r2.toString());
return false;
}
return true;
}
// Split the passed block of text into an array of lines, replacing any
// filenames and line numbers with generic markers to avoid failing the test
// due to compiler-specific behaviour.
static QList<QByteArray> splitLines(QByteArray ba)
{
ba.replace('\r', "");
QList<QByteArray> out = ba.split('\n');
// Replace any ` file="..."' or ` line="..."' in XML with a generic location.
static const char *markers[][2] = {
{ " file=\"", " file=\"__FILE__\"" },
{ " line=\"", " line=\"__LINE__\"" }
};
static const int markerCount = sizeof markers / sizeof markers[0];
for (int i = 0; i < out.size(); ++i) {
QByteArray& line = out[i];
for (int j = 0; j < markerCount; ++j) {
int index = line.indexOf(markers[j][0]);
if (index == -1) {
continue;
}
const int end = line.indexOf('"', index + int(strlen(markers[j][0])));
if (end == -1) {
continue;
}
line.replace(index, end-index + 1, markers[j][1]);
}
}
return out;
}
// Return the log format, e.g. for both "stdout txt" and "txt", return "txt'.
static inline QString logFormat(const QString &logger)
{
return (logger.startsWith("stdout") ? logger.mid(7) : logger);
}
// Return the log file name, or an empty string if the log goes to stdout.
QString tst_Selftests::logName(const QString &logger) const
{
return (logger.startsWith("stdout") ? "" : QString(tempDir.path() + "/test_output." + logger));
}
static QString expectedFileNameFromTest(const QString &subdir, const QString &logger)
{
return QStringLiteral("expected_") + subdir + QLatin1Char('.') + logFormat(logger);
}
// Load the expected test output for the nominated test (subdir) and logger
// as an array of lines. If there is no expected output file, return an
// empty array.
static QList<QByteArray> expectedResult(const QString &fileName)
{
QFile file(QStringLiteral(":/") + fileName);
if (!file.open(QIODevice::ReadOnly))
return QList<QByteArray>();
return splitLines(file.readAll());
}
// Helpers for running the 'diff' tool in case comparison fails
#ifdef USE_DIFF
static inline void writeLines(QIODevice &d, const QByteArrayList &lines)
{
for (const QByteArray &l : lines) {
d.write(l);
d.write("\n");
}
}
#endif // USE_DIFF
static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &actual)
{
QByteArray result;
#ifdef USE_DIFF
# ifndef Q_OS_WIN
const QString diff = QStandardPaths::findExecutable("diff");
# else
const QString diff = QStandardPaths::findExecutable("diff.exe");
# endif
if (diff.isEmpty())
return result;
QTemporaryFile expectedFile;
if (!expectedFile.open())
return result;
writeLines(expectedFile, expected);
expectedFile.close();
QTemporaryFile actualFile;
if (!actualFile.open())
return result;
writeLines(actualFile, actual);
actualFile.close();
QProcess diffProcess;
diffProcess.start(diff, {QLatin1String("-u"), expectedFile.fileName(), actualFile.fileName()});
if (!diffProcess.waitForStarted())
return result;
if (diffProcess.waitForFinished())
result = diffProcess.readAllStandardOutput();
else
diffProcess.kill();
#endif // USE_DIFF
return result;
}
// Each test is run with a set of one or more test output loggers.
// This struct holds information about one such test.
struct LoggerSet
{
LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments)
: name(_name), loggers(_loggers), arguments(_arguments)
{ }
QString name;
QStringList loggers;
QStringList arguments;
};
// This function returns a list of all sets of loggers to be used for
// running each subtest.
QList<LoggerSet> tst_Selftests::allLoggerSets() const
{
// Note that in order to test XML output to standard output, the subtests
// must not send output directly to stdout, bypassing Qt's output mechanisms
// (e.g. via printf), otherwise the output may not be well-formed XML.
return QList<LoggerSet>()
// Test with old-style options for a single logger
<< LoggerSet("old stdout txt",
QStringList() << "stdout txt",
QStringList()
)
<< LoggerSet("old txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt")
)
<< LoggerSet("old stdout xml",
QStringList() << "stdout xml",
QStringList() << "-xml"
)
<< LoggerSet("old xml",
QStringList() << "xml",
QStringList() << "-xml" << "-o" << logName("xml")
)
<< LoggerSet("old stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-xunitxml"
)
<< LoggerSet("old xunitxml",
QStringList() << "xunitxml",
QStringList() << "-xunitxml" << "-o" << logName("xunitxml")
)
<< LoggerSet("old stdout lightxml",
QStringList() << "stdout lightxml",
QStringList() << "-lightxml"
)
<< LoggerSet("old lightxml",
QStringList() << "lightxml",
QStringList() << "-lightxml" << "-o" << logName("lightxml")
)
<< LoggerSet("old stdout csv", // benchmarks only
QStringList() << "stdout csv",
QStringList() << "-csv")
<< LoggerSet("old csv", // benchmarks only
QStringList() << "csv",
QStringList() << "-csv" << "-o" << logName("csv"))
<< LoggerSet("old stdout teamcity",
QStringList() << "stdout teamcity",
QStringList() << "-teamcity"
)
<< LoggerSet("old teamcity",
QStringList() << "teamcity",
QStringList() << "-teamcity" << "-o" << logName("teamcity")
)
<< LoggerSet("old stdout tap",
QStringList() << "stdout tap",
QStringList() << "-tap"
)
<< LoggerSet("old tap",
QStringList() << "tap",
QStringList() << "-tap" << "-o" << logName("tap")
)
// Test with new-style options for a single logger
<< LoggerSet("new stdout txt",
QStringList() << "stdout txt",
QStringList() << "-o" << "-,txt"
)
<< LoggerSet("new txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt")+",txt"
)
<< LoggerSet("new stdout xml",
QStringList() << "stdout xml",
QStringList() << "-o" << "-,xml"
)
<< LoggerSet("new xml",
QStringList() << "xml",
QStringList() << "-o" << logName("xml")+",xml"
)
<< LoggerSet("new stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-o" << "-,xunitxml"
)
<< LoggerSet("new xunitxml",
QStringList() << "xunitxml",
QStringList() << "-o" << logName("xunitxml")+",xunitxml"
)
<< LoggerSet("new stdout lightxml",
QStringList() << "stdout lightxml",
QStringList() << "-o" << "-,lightxml"
)
<< LoggerSet("new lightxml",
QStringList() << "lightxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
)
<< LoggerSet("new stdout csv", // benchmarks only
QStringList() << "stdout csv",
QStringList() << "-o" << "-,csv")
<< LoggerSet("new csv", // benchmarks only
QStringList() << "csv",
QStringList() << "-o" << logName("csv")+",csv")
<< LoggerSet("new stdout teamcity",
QStringList() << "stdout teamcity",
QStringList() << "-o" << "-,teamcity"
)
<< LoggerSet("new teamcity",
QStringList() << "teamcity",
QStringList() << "-o" << logName("teamcity")+",teamcity"
)
<< LoggerSet("new stdout tap",
QStringList() << "stdout tap",
QStringList() << "-o" << "-,tap"
)
<< LoggerSet("new tap",
QStringList() << "tap",
QStringList() << "-o" << logName("tap")+",tap"
)
// Test with two loggers (don't test all 32 combinations, just a sample)
<< LoggerSet("stdout txt + txt",
QStringList() << "stdout txt" << "txt",
QStringList() << "-o" << "-,txt"
<< "-o" << logName("txt")+",txt"
)
<< LoggerSet("xml + stdout txt",
QStringList() << "xml" << "stdout txt",
QStringList() << "-o" << logName("xml")+",xml"
<< "-o" << "-,txt"
)
<< LoggerSet("txt + xunitxml",
QStringList() << "txt" << "xunitxml",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xunitxml")+",xunitxml"
)
<< LoggerSet("lightxml + stdout xunitxml",
QStringList() << "lightxml" << "stdout xunitxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,xunitxml"
)
// All loggers at the same time (except csv)
<< LoggerSet("all loggers",
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml" << "tap",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xml")+",xml"
<< "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,txt"
<< "-o" << logName("xunitxml")+",xunitxml"
<< "-o" << logName("teamcity")+",teamcity"
<< "-o" << logName("tap")+",tap"
)
;
}
tst_Selftests::tst_Selftests()
: tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
, durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>")
, teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]")
{}
void tst_Selftests::initTestCase()
{
QVERIFY2(tempDir.isValid(), qPrintable(tempDir.errorString()));
//Detect the location of the sub programs
QString subProgram = QLatin1String("float/float");
#if defined(Q_OS_WIN)
subProgram = QLatin1String("float/float.exe");
#endif
QString testdataDir = QFINDTESTDATA(subProgram);
if (testdataDir.lastIndexOf(subProgram) > 0)
testdataDir = testdataDir.left(testdataDir.lastIndexOf(subProgram));
else
testdataDir = QCoreApplication::applicationDirPath();
// chdir to our testdata path and execute helper apps relative to that.
QVERIFY2(QDir::setCurrent(testdataDir), qPrintable("Could not chdir to " + testdataDir));
}
void tst_Selftests::runSubTest_data()
{
QTest::addColumn<QString>("subdir");
QTest::addColumn<QStringList>("loggers");
QTest::addColumn<QStringList>("arguments");
QTest::addColumn<bool>("crashes");
QStringList tests = QStringList()
// << "alive" // timer dependent
#if !defined(Q_OS_WIN)
// On windows, assert does nothing in release mode and blocks execution
// with a popup window in debug mode.
<< "assert"
#endif
<< "badxml"
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
// Only run on platforms where callgrind is available.
<< "benchlibcallgrind"
#endif
<< "benchlibcounting"
<< "benchlibeventcounter"
<< "benchliboptions"
<< "blacklisted"
<< "cmptest"
<< "commandlinedata"
<< "counting"
<< "crashes"
<< "datatable"
<< "datetime"
<< "differentexec"
#if !defined(QT_NO_EXCEPTIONS) && !defined(Q_CC_INTEL) && !defined(Q_OS_WIN)
// Disable this test on Windows and for intel compiler, as the run-times
// will popup dialogs with warnings that uncaught exceptions were thrown
<< "exceptionthrow"
#endif
<< "expectfail"
<< "failcleanup"
<< "failinit"
<< "failinitdata"
#if !defined(Q_OS_WIN)
// Disable this test on Windows, as the run-time will popup dialogs with warnings
<< "fetchbogus"
#endif
<< "findtestdata"
<< "float"
<< "globaldata"
<< "keyboard"
<< "longstring"
<< "maxwarnings"
<< "multiexec"
<< "pairdiagnostics"
<< "printdatatags"
<< "printdatatagswithglobaltags"
<< "qexecstringlist"
<< "silent"
<< "singleskip"
<< "skip"
<< "skipcleanup"
<< "skipinit"
<< "skipinitdata"
<< "sleep"
<< "strcmp"
<< "subtest"
<< "tuplediagnostics"
<< "verbose1"
<< "verbose2"
#ifndef QT_NO_EXCEPTIONS
// this test will test nothing if the exceptions are disabled
<< "verifyexceptionthrown"
#endif //!QT_NO_EXCEPTIONS
<< "warnings"
<< "xunit"
;
// These tests are affected by timing and whether the CPU tick counter
// is monotonically increasing. They won't work on some machines so
// leave them off by default. Feel free to enable them for your own
// testing by setting the QTEST_ENABLE_EXTRA_SELFTESTS environment
// variable to something non-empty.
if (!qgetenv("QTEST_ENABLE_EXTRA_SELFTESTS").isEmpty())
tests << "benchlibtickcounter"
<< "benchlibwalltime"
;
foreach (LoggerSet const& loggerSet, allLoggerSets()) {
QStringList loggers = loggerSet.loggers;
foreach (QString const& subtest, tests) {
QStringList arguments = loggerSet.arguments;
if (subtest == "commandlinedata") {
arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
}
else if (subtest == "benchlibcallgrind") {
arguments << "-callgrind";
}
else if (subtest == "benchlibeventcounter") {
arguments << "-eventcounter";
}
else if (subtest == "benchliboptions") {
arguments << "-eventcounter";
}
else if (subtest == "benchlibtickcounter") {
arguments << "-tickcounter";
}
else if (subtest == "badxml") {
arguments << "-eventcounter";
}
else if (subtest == "benchlibcounting") {
arguments << "-eventcounter";
}
else if (subtest == "printdatatags") {
arguments << "-datatags";
}
else if (subtest == "printdatatagswithglobaltags") {
arguments << "-datatags";
}
else if (subtest == "silent") {
arguments << "-silent";
}
else if (subtest == "verbose1") {
arguments << "-v1";
}
else if (subtest == "verbose2") {
arguments << "-v2";
}
// These tests don't work right unless logging plain text to
// standard output, either because they execute multiple test
// objects or because they internally supply arguments to
// themselves.
if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") {
if (subtest == "differentexec") {
continue;
}
if (subtest == "multiexec") {
continue;
}
if (subtest == "qexecstringlist") {
continue;
}
if (subtest == "benchliboptions") {
continue;
}
if (subtest == "blacklisted") {
continue;
}
if (subtest == "printdatatags") {
continue;
}
if (subtest == "printdatatagswithglobaltags") {
continue;
}
if (subtest == "silent") {
continue;
}
// `crashes' will not output valid XML on platforms without a crash handler
if (subtest == "crashes") {
continue;
}
// this test prints out some floats in the testlog and the formatting is
// platform-specific and hard to predict.
if (subtest == "float") {
continue;
}
// these tests are quite slow, and running them for all the loggers significantly
// increases the overall test time. They do not really relate to logging, so it
// should be safe to run them just for the stdout loggers.
if (subtest == "benchlibcallgrind" || subtest == "sleep") {
continue;
}
}
if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt")))
continue; // XML only, do not mix txt and XML for encoding test.
if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib"))
continue;
if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib"))
continue; // Skip benchmark for TeamCity logger
const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
|| subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
|| subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
|| subtest == QLatin1String("blacklisted");
QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
<< subtest
<< loggers
<< arguments
<< crashes
;
}
}
}
#if QT_CONFIG(process)
static QProcessEnvironment processEnvironment()
{
static QProcessEnvironment result;
if (result.isEmpty()) {
const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH");
foreach (const QString &key, systemEnvironment.keys()) {
const bool useVariable = key == QLatin1String("PATH") || key == QLatin1String("QT_QPA_PLATFORM")
#if defined(Q_OS_QNX)
|| key == QLatin1String("GRAPHICS_ROOT") || key == QLatin1String("TZ")
#elif defined(Q_OS_UNIX)
|| key == QLatin1String("HOME") || key == QLatin1String("USER") // Required for X11 on openSUSE
|| key == QLatin1String("QEMU_SET_ENV") || key == QLatin1String("QEMU_LD_PREFIX") // Required for QEMU
# if !defined(Q_OS_MAC)
|| key == QLatin1String("DISPLAY") || key == QLatin1String("XAUTHLOCALHOSTNAME")
|| key.startsWith(QLatin1String("XDG_"))
# endif // !Q_OS_MAC
#endif // Q_OS_UNIX
#ifdef __COVERAGESCANNER__
|| key == QLatin1String("QT_TESTCOCOON_ACTIVE")
#endif
|| ( preserveLibPath && (key == QLatin1String("QT_PLUGIN_PATH")
|| key == QLatin1String("LD_LIBRARY_PATH")))
;
if (useVariable)
result.insert(key, systemEnvironment.value(key));
}
// Avoid interference from any qtlogging.ini files, e.g. in /etc/xdg/QtProject/:
result.insert(QStringLiteral("QT_LOGGING_RULES"),
// Must match generate_expected_output.py's main()'s value:
QStringLiteral("*.debug=true;qt.*=false"));
}
return result;
}
static inline QByteArray msgProcessError(const QString &binary, const QStringList &args,
const QProcessEnvironment &e, const QString &what)
{
QString result;
QTextStream(&result) <<"Error running " << binary << ' ' << args.join(' ')
<< " with environment " << e.toStringList().join(' ') << ": " << what;
return result.toLocal8Bit();
}
void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes)
{
if (EmulationDetector::isRunningArmOnX86() && (subdir == "crashes"))
QSKIP("Skipping \"crashes\" due to QTBUG-71915");
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
if (arguments.contains("-callgrind")) {
QProcess checkProcess;
QStringList args;
args << QLatin1String("--version");
checkProcess.start(QLatin1String("valgrind"), args);
if (!checkProcess.waitForFinished(-1))
QSKIP(QString("Valgrind broken or not available. Not running %1 test!").arg(subdir).toLocal8Bit());
}
#endif
QProcess proc;
QProcessEnvironment environment = processEnvironment();
if (crashes) {
environment.insert("QTEST_DISABLE_CORE_DUMP", "1");
environment.insert("QTEST_DISABLE_STACK_DUMP", "1");
}
proc.setProcessEnvironment(environment);
const QString path = subdir + QLatin1Char('/') + subdir;
proc.start(path, arguments);
QVERIFY2(proc.waitForStarted(), msgProcessError(path, arguments, environment, QStringLiteral("Cannot start: ") + proc.errorString()));
QVERIFY2(proc.waitForFinished(), msgProcessError(path, arguments, environment, QStringLiteral("Timed out: ") + proc.errorString()));
if (!crashes) {
QVERIFY2(proc.exitStatus() == QProcess::NormalExit,
msgProcessError(path, arguments, environment,
QStringLiteral("Crashed: ") + proc.errorString()
+ QStringLiteral(": ") + QString::fromLocal8Bit(proc.readAllStandardError())));
}
QList<QByteArray> actualOutputs;
for (int i = 0; i < loggers.count(); ++i) {
QString logFile = logName(loggers[i]);
QByteArray out;
if (logFile.isEmpty()) {
out = proc.readAllStandardOutput();
} else {
QFile file(logFile);
if (file.open(QIODevice::ReadOnly))
out = file.readAll();
}
actualOutputs << out;
}
const QByteArray err(proc.readAllStandardError());
// Some tests may output unpredictable strings to stderr, which we'll ignore.
//
// For instance, uncaught exceptions on Windows might say (depending on Windows
// version and JIT debugger settings):
// "This application has requested the Runtime to terminate it in an unusual way.
// Please contact the application's support team for more information."
//
// Also, tests which use valgrind may generate warnings if the toolchain is
// newer than the valgrind version, such that valgrind can't understand the
// debug information on the binary.
if (subdir != QLatin1String("exceptionthrow")
&& subdir != QLatin1String("cmptest") // QImage comparison requires QGuiApplication
&& subdir != QLatin1String("fetchbogus")
&& subdir != QLatin1String("xunit")
#ifdef Q_CC_MINGW
&& subdir != QLatin1String("blacklisted") // calls qFatal()
&& subdir != QLatin1String("silent") // calls qFatal()
#endif
#ifdef Q_OS_LINUX
// QEMU outputs to stderr about uncaught signals
&& (!EmulationDetector::isRunningArmOnX86() ||
(subdir != QLatin1String("blacklisted")
&& subdir != QLatin1String("silent")
&& subdir != QLatin1String("assert")
&& subdir != QLatin1String("crashes")
)
)
#endif
&& subdir != QLatin1String("benchlibcallgrind"))
QVERIFY2(err.isEmpty(), err.constData());
for (int n = 0; n < loggers.count(); ++n) {
QString logger = loggers[n];
if (n == 0 && subdir == QLatin1String("crashes")) {
QByteArray &actual = actualOutputs[0];
#ifndef Q_OS_WIN
// Remove digits of times to match the expected file.
const QLatin1String timePattern("Function time:");
int timePos = actual.indexOf(timePattern);
if (timePos >= 0) {
timePos += timePattern.size();
const int nextLinePos = actual.indexOf('\n', timePos);
for (int c = (nextLinePos != -1 ? nextLinePos : actual.size()) - 1; c >= timePos; --c) {
if (actual.at(c) >= '0' && actual.at(c) <= '9')
actual.remove(c, 1);
}
}
#else // !Q_OS_WIN
// Remove stack trace which is output to stdout.
const int exceptionLogStart = actual.indexOf("A crash occurred in ");
if (exceptionLogStart >= 0)
actual.truncate(exceptionLogStart);
#endif // Q_OS_WIN
}
QList<QByteArray> res = splitLines(actualOutputs[n]);
QString errorMessage;
QString expectedFileName = expectedFileNameFromTest(subdir, logger);
QByteArrayList exp = expectedResult(expectedFileName);
if (!exp.isEmpty()) {
if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) {
errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ")
+ expectedFileName + QLatin1Char(' '));
errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]);
const QByteArray diff = runDiff(exp, res);
if (!diff.isEmpty())
errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff);
QFAIL(qPrintable(errorMessage));
}
} else {
// For the "crashes" and other tests, there are multiple versions of the
// expected output. Loop until a matching one is found.
bool ok = false;
for (int i = 1; !ok; ++i) {
expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger);
const QByteArrayList exp = expectedResult(expectedFileName);
if (exp.isEmpty())
break;
QString errorMessage2;
ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2);
if (!ok)
errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2;
}
if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output.
errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir);
errorMessage += QLatin1String("\nActual:\n");
QDebug(&errorMessage) << actualOutputs[n];
QFAIL(qPrintable(errorMessage));
}
}
}
}
static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }
bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
const QByteArray &rawOutput, const QByteArrayList &actual,
const QByteArrayList &expected,
QString *errorMessage) const
{
if (actual.size() != expected.size()) {
*errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.")
.arg(actual.size()).arg(expected.size());
return false;
}
// For xml output formats, verify that the log is valid XML.
if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
return false;
// Verify that the actual output is an acceptable match for the
// expected output.
const QString qtVersion = QLatin1String(QT_VERSION_STR);
bool benchmark = false;
for (int i = 0, size = actual.size(); i < size; ++i) {
const QByteArray &actualLineBA = actual.at(i);
// the __FILE__ __LINE__ output is compiler dependent, skip it
if (actualLineBA.startsWith(" Loc: [") && actualLineBA.endsWith(")]"))
continue;
if (actualLineBA.endsWith(" : failure location"))
continue;
if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string
|| actualLineBA.startsWith(" <QtBuild") // XML, Light XML build string
|| (actualLineBA.startsWith(" <property value=") && actualLineBA.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
continue;
}
QString actualLine = QString::fromLatin1(actualLineBA);
QString expectedLine = QString::fromLatin1(expected.at(i));
expectedLine.replace(qtVersionPlaceHolder(), qtVersion);
// Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
if (logger.endsWith(QLatin1String("teamcity"))) {
actualLine.replace(teamcityLocRegExp, teamCityLocation());
expectedLine.replace(teamcityLocRegExp, teamCityLocation());
}
if (logger.endsWith(QLatin1String("tap"))) {
if (expectedLine.contains(QLatin1String("at:"))
|| expectedLine.contains(QLatin1String("file:"))
|| expectedLine.contains(QLatin1String("line:")))
actualLine = expectedLine;
}
if (!compareLine(logger, subdir, benchmark, actualLine,
expectedLine, errorMessage)) {
errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)
+ QLatin1String(": "));
return false;
}
benchmark = actualLineBA.startsWith("RESULT : ");
}
return true;
}
bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
bool benchmark,
const QString &actualLine, const QString &expectedLine,
QString *errorMessage) const
{
if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: "))
&& expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) {
// Q_ASSERT uses __FILE__, the exact contents of which are
// undefined. If have we something that looks like a Q_ASSERT and we
// were expecting to see a Q_ASSERT, we'll skip the line.
return true;
}
if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce"))
&& actualLine != expectedLine) {
// On some platforms we compile without RTTI, and as a result we never throw an exception
if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
*errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine);
return false;
}
return true;
}
if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult"))
|| (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) {
// Don't do a literal comparison for benchmark results, since
// results have some natural variance.
QString error;
BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
if (!error.isEmpty()) {
*errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine);
return false;
}
BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
if (!error.isEmpty()) {
*errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine);
return false;
}
return compareBenchmarkResult(actualResult, expectedResult, errorMessage);
}
if (actualLine.startsWith(QLatin1String(" <Duration msecs="))
|| actualLine.startsWith(QLatin1String("<Duration msecs="))) {
QRegularExpressionMatch match = durationRegExp.match(actualLine);
if (match.hasMatch())
return true;
*errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine);
return false;
}
if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:")))
return true;
if (actualLine == expectedLine)
return true;
*errorMessage = msgMismatch(actualLine, expectedLine);
return false;
}
bool tst_Selftests::checkXml(const QString &logger, QByteArray xml,
QString *errorMessage) const
{
// lightxml intentionally skips the root element, which technically makes it
// not valid XML.
// We'll add that ourselves for the purpose of validation.
if (logger.endsWith(QLatin1String("lightxml"))) {
xml.prepend("<root>");
xml.append("</root>");
}
QXmlStreamReader reader(xml);
while (!reader.atEnd())
reader.readNext();
if (reader.hasError()) {
const int lineNumber = int(reader.lineNumber());
const QByteArray line = xml.split('\n').value(lineNumber - 1);
*errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4")
.arg(lineNumber).arg(reader.columnNumber())
.arg(QString::fromLatin1(line), reader.errorString());
return false;
}
return true;
}
#endif // QT_CONFIG(process)
void tst_Selftests::runSubTest()
{
#if !QT_CONFIG(process)
QSKIP("This test requires QProcess support");
#else
QFETCH(QString, subdir);
QFETCH(QStringList, loggers);
QFETCH(QStringList, arguments);
QFETCH(bool, crashes);
doRunSubTest(subdir, loggers, arguments, crashes);
#endif // QT_CONFIG(process)
}
// attribute must contain ="
QString extractXmlAttribute(const QString &line, const char *attribute)
{
int index = line.indexOf(attribute);
if (index == -1)
return QString();
const int attributeLength = int(strlen(attribute));
const int end = line.indexOf('"', index + attributeLength);
if (end == -1)
return QString();
const QString result = line.mid(index + attributeLength, end - index - attributeLength);
if (result.isEmpty())
return ""; // ensure empty but not null
return result;
}
// Parse line into the BenchmarkResult it represents.
BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
{
if (error) *error = QString();
BenchmarkResult out;
QString remaining = line.trimmed();
if (remaining.isEmpty()) {
if (error) *error = "Line is empty";
return out;
}
if (line.startsWith("<BenchmarkResult ")) {
// XML result
// format:
// <BenchmarkResult metric="$unit" tag="$tag" value="$total" iterations="$iterations" />
if (!line.endsWith("/>")) {
if (error) *error = "unterminated XML";
return out;
}
QString unit = extractXmlAttribute(line, " metric=\"");
QString sTotal = extractXmlAttribute(line, " value=\"");
QString sIterations = extractXmlAttribute(line, " iterations=\"");
if (unit.isNull() || sTotal.isNull() || sIterations.isNull()) {
if (error) *error = "XML snippet did not contain all required values";
return out;
}
bool ok;
double total = sTotal.toDouble(&ok);
if (!ok) {
if (error) *error = sTotal + " is not a valid number";
return out;
}
double iterations = sIterations.toDouble(&ok);
if (!ok) {
if (error) *error = sIterations + " is not a valid number";
return out;
}
out.unit = unit;
out.total = total;
out.iterations = iterations;
return out;
}
if (line.startsWith('"')) {
// CSV result
// format:
// "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
QStringList split = line.split(',');
if (split.count() != 6) {
if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
return out;
}
bool ok;
double total = split.at(4).toDouble(&ok);
if (!ok) {
if (error) *error = split.at(4) + " is not a valid number";
return out;
}
double iterations = split.at(5).toDouble(&ok);
if (!ok) {
if (error) *error = split.at(5) + " is not a valid number";
return out;
}
out.unit = split.at(2);
out.total = total;
out.iterations = iterations;
return out;
}
// Text result
// This code avoids using a QRegExp because QRegExp might be broken.
// Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)
QString sFirstNumber;
while (!remaining.isEmpty() && !remaining.at(0).isSpace()) {
sFirstNumber += remaining.at(0);
remaining.remove(0,1);
}
remaining = remaining.trimmed();
// 4,000 -> 4000
sFirstNumber.remove(',');
// Should now be parseable as floating point
bool ok;
double firstNumber = sFirstNumber.toDouble(&ok);
if (!ok) {
if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";
return out;
}
// Remaining: msec per iteration (total: 4000, iterations: 1)
static const char periterbit[] = " per iteration (total: ";
QString unit;
while (!remaining.startsWith(periterbit) && !remaining.isEmpty()) {
unit += remaining.at(0);
remaining.remove(0,1);
}
if (remaining.isEmpty()) {
if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";
return out;
}
remaining = remaining.mid(sizeof(periterbit)-1);
// Remaining: 4,000, iterations: 1)
static const char itersbit[] = ", iterations: ";
QString sTotal;
while (!remaining.startsWith(itersbit) && !remaining.isEmpty()) {
sTotal += remaining.at(0);
remaining.remove(0,1);
}
if (remaining.isEmpty()) {
if (error) *error = "Could not find pattern: '<number>, iterations: '";
return out;
}
remaining = remaining.mid(sizeof(itersbit)-1);
// 4,000 -> 4000
sTotal.remove(',');
double total = sTotal.toDouble(&ok);
if (!ok) {
if (error) *error = sTotal + " (total) is not a valid number";
return out;
}
// Remaining: 1)
QString sIters;
while (remaining != QLatin1String(")") && !remaining.isEmpty()) {
sIters += remaining.at(0);
remaining.remove(0,1);
}
if (remaining.isEmpty()) {
if (error) *error = "Could not find pattern: '<num>)'";
return out;
}
qint64 iters = sIters.toLongLong(&ok);
if (!ok) {
if (error) *error = sIters + " (iterations) is not a valid integer";
return out;
}
double calcFirstNumber = double(total)/double(iters);
if (!qFuzzyCompare(firstNumber, calcFirstNumber)) {
if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(calcFirstNumber).arg(firstNumber);
return out;
}
out.total = total;
out.unit = unit;
out.iterations = iters;
return out;
}
void tst_Selftests::cleanup()
{
QFETCH(QStringList, loggers);
// Remove the test output files
for (int i = 0; i < loggers.count(); ++i) {
QString logFileName = logName(loggers[i]);
if (!logFileName.isEmpty()) {
QFile logFile(logFileName);
if (logFile.exists())
QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: ").arg(logFileName, logFile.errorString())));
}
}
}
QTEST_MAIN(tst_Selftests)
#include "tst_selftests.moc"
Reference in New Issue View Git Blame Copy Permalink