AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
4b1b9ad7b0
qt5base-lts/tests/auto/testlib/selftests/tst_selftests.cpp
Edward Welbourne 7c7e007794 Add a testlib selftest for the watchdog 
Verify that it does cut in after the specified time has elapsed.

Task-number: QTPM-1385
Change-Id: Ib18e8d6af28339f79cca4d62b869287ce07b8cc1
Reviewed-by: Jędrzej Nowacki <jedrzej.nowacki@qt.io>
2018-12-06 18:29:47 +00:00

1208 lines
46 KiB
C++
Raw Blame History

/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2016 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the test suite of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include <QtCore/QCoreApplication>
#if QT_CONFIG(temporaryfile) && QT_CONFIG(process)
# define USE_DIFF
#endif
#include <QtCore/QXmlStreamReader>
#include <QtCore/QFileInfo>
#include <QtCore/QDir>
#include <QtCore/QTemporaryDir>
#ifdef USE_DIFF
# include <QtCore/QTemporaryFile>
# include <QtCore/QStandardPaths>
#endif
#include <QtTest/QtTest>
#include <private/cycle_p.h>
#include "emulationdetector.h"
struct LoggerSet;
class tst_Selftests: public QObject
{
Q_OBJECT
public:
tst_Selftests();
private slots:
void initTestCase();
void runSubTest_data();
void runSubTest();
void cleanup();
private:
void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes);
bool compareOutput(const QString &logger, const QString &subdir,
const QByteArray &rawOutput, const QByteArrayList &actual,
const QByteArrayList &expected,
QString *errorMessage) const;
bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
const QString &actualLine, const QString &expLine,
QString *errorMessage) const;
bool checkXml(const QString &logger, QByteArray rawOutput,
QString *errorMessage) const;
QString logName(const QString &logger) const;
QList<LoggerSet> allLoggerSets() const;
QTemporaryDir tempDir;
QRegularExpression durationRegExp;
QRegularExpression teamcityLocRegExp;
};
struct BenchmarkResult
{
qint64 total;
qint64 iterations;
QString unit;
inline QString toString() const
{ return QString("total:%1, unit:%2, iterations:%3").arg(total).arg(unit).arg(iterations); }
static BenchmarkResult parse(QString const&, QString*);
};
static QString msgMismatch(const QString &actual, const QString &expected)
{
return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
+ expected + QLatin1Char('\'');
}
static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
QString *errorMessage)
{
// First make sure the iterations and unit match.
if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
// Nope - compare whole string for best failure message
*errorMessage = msgMismatch(r1.toString(), r2.toString());
return false;
}
// Now check the value. Some variance is allowed, and how much depends on
// the measured unit.
qreal variance = 0.;
if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
variance = 0.1;
else if (r1.unit == QLatin1String("instruction reads"))
variance = 0.001;
else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
variance = 0.001;
if (variance == 0.) {
// No variance allowed - compare whole string
const QString r1S = r1.toString();
const QString r2S = r2.toString();
if (r1S != r2S) {
*errorMessage = msgMismatch(r1S, r2S);
return false;
}
return true;
}
if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
// Whoops, didn't match. Compare the whole string for the most useful failure message.
*errorMessage = msgMismatch(r1.toString(), r2.toString());
return false;
}
return true;
}
// Split the passed block of text into an array of lines, replacing any
// filenames and line numbers with generic markers to avoid failing the test
// due to compiler-specific behaviour.
static QList<QByteArray> splitLines(QByteArray ba)
{
ba.replace('\r', "");
QList<QByteArray> out = ba.split('\n');
// Replace any ` file="..."' or ` line="..."' in XML with a generic location.
static const char *markers[][2] = {
{ " file=\"", " file=\"__FILE__\"" },
{ " line=\"", " line=\"__LINE__\"" }
};
static const int markerCount = sizeof markers / sizeof markers[0];
for (int i = 0; i < out.size(); ++i) {
QByteArray& line = out[i];
for (int j = 0; j < markerCount; ++j) {
int index = line.indexOf(markers[j][0]);
if (index == -1) {
continue;
}
const int end = line.indexOf('"', index + int(strlen(markers[j][0])));
if (end == -1) {
continue;
}
line.replace(index, end-index + 1, markers[j][1]);
}
}
return out;
}
// Return the log format, e.g. for both "stdout txt" and "txt", return "txt'.
static inline QString logFormat(const QString &logger)
{
return (logger.startsWith("stdout") ? logger.mid(7) : logger);
}
// Return the log file name, or an empty string if the log goes to stdout.
QString tst_Selftests::logName(const QString &logger) const
{
return (logger.startsWith("stdout") ? "" : QString(tempDir.path() + "/test_output." + logger));
}
static QString expectedFileNameFromTest(const QString &subdir, const QString &logger)
{
return QStringLiteral("expected_") + subdir + QLatin1Char('.') + logFormat(logger);
}
// Load the expected test output for the nominated test (subdir) and logger
// as an array of lines. If there is no expected output file, return an
// empty array.
static QList<QByteArray> expectedResult(const QString &fileName)
{
QFile file(QStringLiteral(":/") + fileName);
if (!file.open(QIODevice::ReadOnly))
return QList<QByteArray>();
return splitLines(file.readAll());
}
// Helpers for running the 'diff' tool in case comparison fails
#ifdef USE_DIFF
static inline void writeLines(QIODevice &d, const QByteArrayList &lines)
{
for (const QByteArray &l : lines) {
d.write(l);
d.write("\n");
}
}
#endif // USE_DIFF
static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &actual)
{
QByteArray result;
#ifdef USE_DIFF
# ifndef Q_OS_WIN
const QString diff = QStandardPaths::findExecutable("diff");
# else
const QString diff = QStandardPaths::findExecutable("diff.exe");
# endif
if (diff.isEmpty())
return result;
QTemporaryFile expectedFile;
if (!expectedFile.open())
return result;
writeLines(expectedFile, expected);
expectedFile.close();
QTemporaryFile actualFile;
if (!actualFile.open())
return result;
writeLines(actualFile, actual);
actualFile.close();
QProcess diffProcess;
diffProcess.start(diff, {QLatin1String("-u"), expectedFile.fileName(), actualFile.fileName()});
if (!diffProcess.waitForStarted())
return result;
if (diffProcess.waitForFinished())
result = diffProcess.readAllStandardOutput();
else
diffProcess.kill();
#endif // USE_DIFF
return result;
}
// Each test is run with a set of one or more test output loggers.
// This struct holds information about one such test.
struct LoggerSet
{
LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments)
: name(_name), loggers(_loggers), arguments(_arguments)
{ }
QString name;
QStringList loggers;
QStringList arguments;
};
// This function returns a list of all sets of loggers to be used for
// running each subtest.
QList<LoggerSet> tst_Selftests::allLoggerSets() const
{
// Note that in order to test XML output to standard output, the subtests
// must not send output directly to stdout, bypassing Qt's output mechanisms
// (e.g. via printf), otherwise the output may not be well-formed XML.
return QList<LoggerSet>()
// Test with old-style options for a single logger
<< LoggerSet("old stdout txt",
QStringList() << "stdout txt",
QStringList()
)
<< LoggerSet("old txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt")
)
<< LoggerSet("old stdout xml",
QStringList() << "stdout xml",
QStringList() << "-xml"
)
<< LoggerSet("old xml",
QStringList() << "xml",
QStringList() << "-xml" << "-o" << logName("xml")
)
<< LoggerSet("old stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-xunitxml"
)
<< LoggerSet("old xunitxml",
QStringList() << "xunitxml",
QStringList() << "-xunitxml" << "-o" << logName("xunitxml")
)
<< LoggerSet("old stdout lightxml",
QStringList() << "stdout lightxml",
QStringList() << "-lightxml"
)
<< LoggerSet("old lightxml",
QStringList() << "lightxml",
QStringList() << "-lightxml" << "-o" << logName("lightxml")
)
<< LoggerSet("old stdout csv", // benchmarks only
QStringList() << "stdout csv",
QStringList() << "-csv")
<< LoggerSet("old csv", // benchmarks only
QStringList() << "csv",
QStringList() << "-csv" << "-o" << logName("csv"))
<< LoggerSet("old stdout teamcity",
QStringList() << "stdout teamcity",
QStringList() << "-teamcity"
)
<< LoggerSet("old teamcity",
QStringList() << "teamcity",
QStringList() << "-teamcity" << "-o" << logName("teamcity")
)
<< LoggerSet("old stdout tap",
QStringList() << "stdout tap",
QStringList() << "-tap"
)
<< LoggerSet("old tap",
QStringList() << "tap",
QStringList() << "-tap" << "-o" << logName("tap")
)
// Test with new-style options for a single logger
<< LoggerSet("new stdout txt",
QStringList() << "stdout txt",
QStringList() << "-o" << "-,txt"
)
<< LoggerSet("new txt",
QStringList() << "txt",
QStringList() << "-o" << logName("txt")+",txt"
)
<< LoggerSet("new stdout xml",
QStringList() << "stdout xml",
QStringList() << "-o" << "-,xml"
)
<< LoggerSet("new xml",
QStringList() << "xml",
QStringList() << "-o" << logName("xml")+",xml"
)
<< LoggerSet("new stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-o" << "-,xunitxml"
)
<< LoggerSet("new xunitxml",
QStringList() << "xunitxml",
QStringList() << "-o" << logName("xunitxml")+",xunitxml"
)
<< LoggerSet("new stdout lightxml",
QStringList() << "stdout lightxml",
QStringList() << "-o" << "-,lightxml"
)
<< LoggerSet("new lightxml",
QStringList() << "lightxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
)
<< LoggerSet("new stdout csv", // benchmarks only
QStringList() << "stdout csv",
QStringList() << "-o" << "-,csv")
<< LoggerSet("new csv", // benchmarks only
QStringList() << "csv",
QStringList() << "-o" << logName("csv")+",csv")
<< LoggerSet("new stdout teamcity",
QStringList() << "stdout teamcity",
QStringList() << "-o" << "-,teamcity"
)
<< LoggerSet("new teamcity",
QStringList() << "teamcity",
QStringList() << "-o" << logName("teamcity")+",teamcity"
)
<< LoggerSet("new stdout tap",
QStringList() << "stdout tap",
QStringList() << "-o" << "-,tap"
)
<< LoggerSet("new tap",
QStringList() << "tap",
QStringList() << "-o" << logName("tap")+",tap"
)
// Test with two loggers (don't test all 32 combinations, just a sample)
<< LoggerSet("stdout txt + txt",
QStringList() << "stdout txt" << "txt",
QStringList() << "-o" << "-,txt"
<< "-o" << logName("txt")+",txt"
)
<< LoggerSet("xml + stdout txt",
QStringList() << "xml" << "stdout txt",
QStringList() << "-o" << logName("xml")+",xml"
<< "-o" << "-,txt"
)
<< LoggerSet("txt + xunitxml",
QStringList() << "txt" << "xunitxml",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xunitxml")+",xunitxml"
)
<< LoggerSet("lightxml + stdout xunitxml",
QStringList() << "lightxml" << "stdout xunitxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,xunitxml"
)
// All loggers at the same time (except csv)
<< LoggerSet("all loggers",
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml" << "tap",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xml")+",xml"
<< "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,txt"
<< "-o" << logName("xunitxml")+",xunitxml"
<< "-o" << logName("teamcity")+",teamcity"
<< "-o" << logName("tap")+",tap"
)
;
}
tst_Selftests::tst_Selftests()
: tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
, durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>")
, teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]")
{}
void tst_Selftests::initTestCase()
{
QVERIFY2(tempDir.isValid(), qPrintable(tempDir.errorString()));
//Detect the location of the sub programs
QString subProgram = QLatin1String("float/float");
#if defined(Q_OS_WIN)
subProgram = QLatin1String("float/float.exe");
#endif
QString testdataDir = QFINDTESTDATA(subProgram);
if (testdataDir.lastIndexOf(subProgram) > 0)
testdataDir = testdataDir.left(testdataDir.lastIndexOf(subProgram));
else
testdataDir = QCoreApplication::applicationDirPath();
// chdir to our testdata path and execute helper apps relative to that.
QVERIFY2(QDir::setCurrent(testdataDir), qPrintable("Could not chdir to " + testdataDir));
}
void tst_Selftests::runSubTest_data()
{
QTest::addColumn<QString>("subdir");
QTest::addColumn<QStringList>("loggers");
QTest::addColumn<QStringList>("arguments");
QTest::addColumn<bool>("crashes");
QStringList tests = QStringList()
// << "alive" // timer dependent
#if !defined(Q_OS_WIN)
// On windows, assert does nothing in release mode and blocks execution
// with a popup window in debug mode.
<< "assert"
#endif
<< "badxml"
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
// Only run on platforms where callgrind is available.
<< "benchlibcallgrind"
#endif
<< "benchlibcounting"
<< "benchlibeventcounter"
<< "benchliboptions"
<< "blacklisted"
<< "cmptest"
<< "commandlinedata"
<< "counting"
<< "crashes"
<< "datatable"
<< "datetime"
<< "differentexec"
#if !defined(QT_NO_EXCEPTIONS) && !defined(Q_CC_INTEL) && !defined(Q_OS_WIN)
// Disable this test on Windows and for intel compiler, as the run-times
// will popup dialogs with warnings that uncaught exceptions were thrown
<< "exceptionthrow"
#endif
<< "expectfail"
<< "failcleanup"
#ifndef Q_OS_WIN // these assert, by design; so same problem as "assert"
<< "faildatatype"
<< "failfetchtype"
#endif
<< "failinit"
<< "failinitdata"
#ifndef Q_OS_WIN // asserts, by design; so same problem as "assert"
<< "fetchbogus"
#endif
<< "findtestdata"
<< "float"
<< "globaldata"
<< "keyboard"
<< "longstring"
<< "maxwarnings"
<< "multiexec"
<< "pairdiagnostics"
<< "printdatatags"
<< "printdatatagswithglobaltags"
<< "qexecstringlist"
<< "silent"
<< "singleskip"
<< "skip"
<< "skipcleanup"
<< "skipinit"
<< "skipinitdata"
<< "sleep"
<< "strcmp"
<< "subtest"
<< "tuplediagnostics"
<< "verbose1"
<< "verbose2"
#ifndef QT_NO_EXCEPTIONS
// this test will test nothing if the exceptions are disabled
<< "verifyexceptionthrown"
#endif //!QT_NO_EXCEPTIONS
<< "warnings"
<< "watchdog"
<< "xunit"
;
// These tests are affected by timing and whether the CPU tick counter
// is monotonically increasing. They won't work on some machines so
// leave them off by default. Feel free to enable them for your own
// testing by setting the QTEST_ENABLE_EXTRA_SELFTESTS environment
// variable to something non-empty.
if (!qgetenv("QTEST_ENABLE_EXTRA_SELFTESTS").isEmpty())
tests << "benchlibtickcounter"
<< "benchlibwalltime"
;
foreach (LoggerSet const& loggerSet, allLoggerSets()) {
QStringList loggers = loggerSet.loggers;
foreach (QString const& subtest, tests) {
QStringList arguments = loggerSet.arguments;
// Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py:
if (subtest == "commandlinedata") {
arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
}
else if (subtest == "benchlibcallgrind") {
arguments << "-callgrind";
}
else if (subtest == "benchlibeventcounter") {
arguments << "-eventcounter";
}
else if (subtest == "benchliboptions") {
arguments << "-eventcounter";
}
else if (subtest == "benchlibtickcounter") {
arguments << "-tickcounter";
}
else if (subtest == "badxml") {
arguments << "-eventcounter";
}
else if (subtest == "benchlibcounting") {
arguments << "-eventcounter";
}
else if (subtest == "printdatatags") {
arguments << "-datatags";
}
else if (subtest == "printdatatagswithglobaltags") {
arguments << "-datatags";
}
else if (subtest == "silent") {
arguments << "-silent";
}
else if (subtest == "verbose1") {
arguments << "-v1";
}
else if (subtest == "verbose2") {
arguments << "-v2";
}
// These tests don't work right unless logging plain text to
// standard output, either because they execute multiple test
// objects or because they internally supply arguments to
// themselves.
if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") {
if (subtest == "differentexec") {
continue;
}
if (subtest == "multiexec") {
continue;
}
if (subtest == "qexecstringlist") {
continue;
}
if (subtest == "benchliboptions") {
continue;
}
if (subtest == "blacklisted") {
continue;
}
if (subtest == "printdatatags") {
continue;
}
if (subtest == "printdatatagswithglobaltags") {
continue;
}
if (subtest == "silent") {
continue;
}
// `crashes' will not output valid XML on platforms without a crash handler
if (subtest == "crashes") {
continue;
}
// this test prints out some floats in the testlog and the formatting is
// platform-specific and hard to predict.
if (subtest == "float") {
continue;
}
// these tests are quite slow, and running them for all the loggers significantly
// increases the overall test time. They do not really relate to logging, so it
// should be safe to run them just for the stdout loggers.
if (subtest == "benchlibcallgrind" || subtest == "sleep") {
continue;
}
}
if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt")))
continue; // XML only, do not mix txt and XML for encoding test.
if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib"))
continue;
if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib"))
continue; // Skip benchmark for TeamCity logger
// Keep in sync with generateTestData()'s crashers in generate_expected_output.py:
const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
|| subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
|| subtest == QLatin1String("faildatatype") || subtest == QLatin1String("failfetchtype")
|| subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
|| subtest == QLatin1String("blacklisted") || subtest == QLatin1String("watchdog");
QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
<< subtest
<< loggers
<< arguments
<< crashes
;
}
}
}
#if QT_CONFIG(process)
static QProcessEnvironment processEnvironment()
{
static QProcessEnvironment result;
if (result.isEmpty()) {
const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH");
foreach (const QString &key, systemEnvironment.keys()) {
const bool useVariable = key == QLatin1String("PATH") || key == QLatin1String("QT_QPA_PLATFORM")
#if defined(Q_OS_QNX)
|| key == QLatin1String("GRAPHICS_ROOT") || key == QLatin1String("TZ")
#elif defined(Q_OS_UNIX)
|| key == QLatin1String("HOME") || key == QLatin1String("USER") // Required for X11 on openSUSE
|| key == QLatin1String("QEMU_SET_ENV") || key == QLatin1String("QEMU_LD_PREFIX") // Required for QEMU
# if !defined(Q_OS_MAC)
|| key == QLatin1String("DISPLAY") || key == QLatin1String("XAUTHLOCALHOSTNAME")
|| key.startsWith(QLatin1String("XDG_"))
# endif // !Q_OS_MAC
#endif // Q_OS_UNIX
#ifdef __COVERAGESCANNER__
|| key == QLatin1String("QT_TESTCOCOON_ACTIVE")
#endif
|| ( preserveLibPath && (key == QLatin1String("QT_PLUGIN_PATH")
|| key == QLatin1String("LD_LIBRARY_PATH")))
;
if (useVariable)
result.insert(key, systemEnvironment.value(key));
}
// Avoid interference from any qtlogging.ini files, e.g. in /etc/xdg/QtProject/:
result.insert(QStringLiteral("QT_LOGGING_RULES"),
// Must match generate_expected_output.py's main()'s value:
QStringLiteral("*.debug=true;qt.*=false"));
}
return result;
}
static inline QByteArray msgProcessError(const QString &binary, const QStringList &args,
const QProcessEnvironment &e, const QString &what)
{
QString result;
QTextStream(&result) <<"Error running " << binary << ' ' << args.join(' ')
<< " with environment " << e.toStringList().join(' ') << ": " << what;
return result.toLocal8Bit();
}
void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes)
{
if (EmulationDetector::isRunningArmOnX86() && (subdir == "crashes"))
QSKIP("Skipping \"crashes\" due to QTBUG-71915");
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
if (arguments.contains("-callgrind")) {
QProcess checkProcess;
QStringList args;
args << QLatin1String("--version");
checkProcess.start(QLatin1String("valgrind"), args);
if (!checkProcess.waitForFinished(-1))
QSKIP(QString("Valgrind broken or not available. Not running %1 test!").arg(subdir).toLocal8Bit());
}
#endif
QProcess proc;
QProcessEnvironment environment = processEnvironment();
// Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py:
if (crashes) {
environment.insert("QTEST_DISABLE_CORE_DUMP", "1");
environment.insert("QTEST_DISABLE_STACK_DUMP", "1");
if (subdir == QLatin1String("watchdog"))
environment.insert("QTEST_FUNCTION_TIMEOUT", "100");
}
proc.setProcessEnvironment(environment);
const QString path = subdir + QLatin1Char('/') + subdir;
proc.start(path, arguments);
QVERIFY2(proc.waitForStarted(), msgProcessError(path, arguments, environment, QStringLiteral("Cannot start: ") + proc.errorString()));
QVERIFY2(proc.waitForFinished(), msgProcessError(path, arguments, environment, QStringLiteral("Timed out: ") + proc.errorString()));
if (!crashes) {
QVERIFY2(proc.exitStatus() == QProcess::NormalExit,
msgProcessError(path, arguments, environment,
QStringLiteral("Crashed: ") + proc.errorString()
+ QStringLiteral(": ") + QString::fromLocal8Bit(proc.readAllStandardError())));
}
QList<QByteArray> actualOutputs;
for (int i = 0; i < loggers.count(); ++i) {
QString logFile = logName(loggers[i]);
QByteArray out;
if (logFile.isEmpty()) {
out = proc.readAllStandardOutput();
} else {
QFile file(logFile);
if (file.open(QIODevice::ReadOnly))
out = file.readAll();
}
actualOutputs << out;
}
const QByteArray err(proc.readAllStandardError());
// Some tests may output unpredictable strings to stderr, which we'll ignore.
//
// For instance, uncaught exceptions on Windows might say (depending on Windows
// version and JIT debugger settings):
// "This application has requested the Runtime to terminate it in an unusual way.
// Please contact the application's support team for more information."
//
// Also, tests which use valgrind may generate warnings if the toolchain is
// newer than the valgrind version, such that valgrind can't understand the
// debug information on the binary.
if (subdir != QLatin1String("exceptionthrow")
&& subdir != QLatin1String("cmptest") // QImage comparison requires QGuiApplication
&& subdir != QLatin1String("fetchbogus")
&& subdir != QLatin1String("watchdog")
&& subdir != QLatin1String("xunit")
#ifdef Q_CC_MINGW
&& subdir != QLatin1String("blacklisted") // calls qFatal()
&& subdir != QLatin1String("silent") // calls qFatal()
#endif
#ifdef Q_OS_LINUX
// QEMU outputs to stderr about uncaught signals
&& !(EmulationDetector::isRunningArmOnX86() &&
(subdir == QLatin1String("assert")
|| subdir == QLatin1String("blacklisted")
|| subdir == QLatin1String("crashes")
|| subdir == QLatin1String("faildatatype")
|| subdir == QLatin1String("failfetchtype")
|| subdir == QLatin1String("silent")
)
)
#endif
&& subdir != QLatin1String("benchlibcallgrind"))
QVERIFY2(err.isEmpty(), err.constData());
for (int n = 0; n < loggers.count(); ++n) {
QString logger = loggers[n];
if (n == 0 && subdir == QLatin1String("crashes")) {
QByteArray &actual = actualOutputs[0];
#ifndef Q_OS_WIN
// Remove digits of times to match the expected file.
const QLatin1String timePattern("Function time:");
int timePos = actual.indexOf(timePattern);
if (timePos >= 0) {
timePos += timePattern.size();
const int nextLinePos = actual.indexOf('\n', timePos);
for (int c = (nextLinePos != -1 ? nextLinePos : actual.size()) - 1; c >= timePos; --c) {
if (actual.at(c) >= '0' && actual.at(c) <= '9')
actual.remove(c, 1);
}
}
#else // !Q_OS_WIN
// Remove stack trace which is output to stdout.
const int exceptionLogStart = actual.indexOf("A crash occurred in ");
if (exceptionLogStart >= 0)
actual.truncate(exceptionLogStart);
#endif // Q_OS_WIN
}
QList<QByteArray> res = splitLines(actualOutputs[n]);
QString errorMessage;
QString expectedFileName = expectedFileNameFromTest(subdir, logger);
QByteArrayList exp = expectedResult(expectedFileName);
if (!exp.isEmpty()) {
#ifdef Q_CC_MINGW
// MinGW formats double numbers differently (last verified with 7.1)
if (n == 0 && subdir == QStringLiteral("float")) {
for (int i = 0; i < exp.size(); ++i) {
exp[i].replace("e-07", "e-007");
exp[i].replace("e+07", "e+007");
}
}
#endif
if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) {
errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ")
+ expectedFileName + QLatin1Char(' '));
errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]);
const QByteArray diff = runDiff(exp, res);
if (!diff.isEmpty())
errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff);
QFAIL(qPrintable(errorMessage));
}
} else {
// For the "crashes" and other tests, there are multiple versions of the
// expected output. Loop until a matching one is found.
bool ok = false;
for (int i = 1; !ok; ++i) {
expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger);
const QByteArrayList exp = expectedResult(expectedFileName);
if (exp.isEmpty())
break;
QString errorMessage2;
ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2);
if (!ok)
errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2;
}
if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output.
errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir);
errorMessage += QLatin1String("\nActual:\n");
QDebug(&errorMessage) << actualOutputs[n];
QFAIL(qPrintable(errorMessage));
}
}
}
}
static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }
bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
const QByteArray &rawOutput, const QByteArrayList &actual,
const QByteArrayList &expected,
QString *errorMessage) const
{
if (actual.size() != expected.size()) {
*errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.")
.arg(actual.size()).arg(expected.size());
return false;
}
// For xml output formats, verify that the log is valid XML.
if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
return false;
// Verify that the actual output is an acceptable match for the
// expected output.
const QString qtVersion = QLatin1String(QT_VERSION_STR);
bool benchmark = false;
for (int i = 0, size = actual.size(); i < size; ++i) {
const QByteArray &actualLineBA = actual.at(i);
// the __FILE__ __LINE__ output is compiler dependent, skip it
if (actualLineBA.startsWith(" Loc: [") && actualLineBA.endsWith(")]"))
continue;
if (actualLineBA.endsWith(" : failure location"))
continue;
if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string
|| actualLineBA.startsWith(" <QtBuild") // XML, Light XML build string
|| (actualLineBA.startsWith(" <property value=") && actualLineBA.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
continue;
}
QString actualLine = QString::fromLatin1(actualLineBA);
QString expectedLine = QString::fromLatin1(expected.at(i));
expectedLine.replace(qtVersionPlaceHolder(), qtVersion);
// Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
if (logger.endsWith(QLatin1String("teamcity"))) {
actualLine.replace(teamcityLocRegExp, teamCityLocation());
expectedLine.replace(teamcityLocRegExp, teamCityLocation());
}
if (logger.endsWith(QLatin1String("tap"))) {
if (expectedLine.contains(QLatin1String("at:"))
|| expectedLine.contains(QLatin1String("file:"))
|| expectedLine.contains(QLatin1String("line:")))
actualLine = expectedLine;
}
if (!compareLine(logger, subdir, benchmark, actualLine,
expectedLine, errorMessage)) {
errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)
+ QLatin1String(": "));
return false;
}
benchmark = actualLineBA.startsWith("RESULT : ");
}
return true;
}
bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
bool benchmark,
const QString &actualLine, const QString &expectedLine,
QString *errorMessage) const
{
if (actualLine == expectedLine)
return true;
if ((subdir == QLatin1String("assert")
|| subdir == QLatin1String("faildatatype") || subdir == QLatin1String("failfetchtype"))
&& actualLine.contains(QLatin1String("ASSERT: "))
&& expectedLine.contains(QLatin1String("ASSERT: "))) {
// Q_ASSERT uses __FILE__, the exact contents of which are
// undefined. If have we something that looks like a Q_ASSERT and we
// were expecting to see a Q_ASSERT, we'll skip the line.
return true;
}
if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce"))) {
// On some platforms we compile without RTTI, and as a result we never throw an exception
if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
*errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine);
return false;
}
return true;
}
if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult"))
|| (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) {
// Don't do a literal comparison for benchmark results, since
// results have some natural variance.
QString error;
BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
if (!error.isEmpty()) {
*errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine);
return false;
}
BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
if (!error.isEmpty()) {
*errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine);
return false;
}
return compareBenchmarkResult(actualResult, expectedResult, errorMessage);
}
if (actualLine.startsWith(QLatin1String(" <Duration msecs="))
|| actualLine.startsWith(QLatin1String("<Duration msecs="))) {
QRegularExpressionMatch match = durationRegExp.match(actualLine);
if (match.hasMatch())
return true;
*errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine);
return false;
}
if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:")))
return true;
*errorMessage = msgMismatch(actualLine, expectedLine);
return false;
}
bool tst_Selftests::checkXml(const QString &logger, QByteArray xml,
QString *errorMessage) const
{
// lightxml intentionally skips the root element, which technically makes it
// not valid XML.
// We'll add that ourselves for the purpose of validation.
if (logger.endsWith(QLatin1String("lightxml"))) {
xml.prepend("<root>");
xml.append("</root>");
}
QXmlStreamReader reader(xml);
while (!reader.atEnd())
reader.readNext();
if (reader.hasError()) {
const int lineNumber = int(reader.lineNumber());
const QByteArray line = xml.split('\n').value(lineNumber - 1);
*errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4")
.arg(lineNumber).arg(reader.columnNumber())
.arg(QString::fromLatin1(line), reader.errorString());
return false;
}
return true;
}
#endif // QT_CONFIG(process)
void tst_Selftests::runSubTest()
{
#if !QT_CONFIG(process)
QSKIP("This test requires QProcess support");
#else
QFETCH(QString, subdir);
QFETCH(QStringList, loggers);
QFETCH(QStringList, arguments);
QFETCH(bool, crashes);
doRunSubTest(subdir, loggers, arguments, crashes);
#endif // QT_CONFIG(process)
}
// attribute must contain ="
QString extractXmlAttribute(const QString &line, const char *attribute)
{
int index = line.indexOf(attribute);
if (index == -1)
return QString();
const int attributeLength = int(strlen(attribute));
const int end = line.indexOf('"', index + attributeLength);
if (end == -1)
return QString();
const QString result = line.mid(index + attributeLength, end - index - attributeLength);
if (result.isEmpty())
return ""; // ensure empty but not null
return result;
}
// Parse line into the BenchmarkResult it represents.
BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
{
if (error) *error = QString();
BenchmarkResult out;
QString remaining = line.trimmed();
if (remaining.isEmpty()) {
if (error) *error = "Line is empty";
return out;
}
if (line.startsWith("<BenchmarkResult ")) {
// XML result
// format:
// <BenchmarkResult metric="$unit" tag="$tag" value="$total" iterations="$iterations" />
if (!line.endsWith("/>")) {
if (error) *error = "unterminated XML";
return out;
}
QString unit = extractXmlAttribute(line, " metric=\"");
QString sTotal = extractXmlAttribute(line, " value=\"");
QString sIterations = extractXmlAttribute(line, " iterations=\"");
if (unit.isNull() || sTotal.isNull() || sIterations.isNull()) {
if (error) *error = "XML snippet did not contain all required values";
return out;
}
bool ok;
double total = sTotal.toDouble(&ok);
if (!ok) {
if (error) *error = sTotal + " is not a valid number";
return out;
}
double iterations = sIterations.toDouble(&ok);
if (!ok) {
if (error) *error = sIterations + " is not a valid number";
return out;
}
out.unit = unit;
out.total = total;
out.iterations = iterations;
return out;
}
if (line.startsWith('"')) {
// CSV result
// format:
// "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
QStringList split = line.split(',');
if (split.count() != 6) {
if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
return out;
}
bool ok;
double total = split.at(4).toDouble(&ok);
if (!ok) {
if (error) *error = split.at(4) + " is not a valid number";
return out;
}
double iterations = split.at(5).toDouble(&ok);
if (!ok) {
if (error) *error = split.at(5) + " is not a valid number";
return out;
}
out.unit = split.at(2);
out.total = total;
out.iterations = iterations;
return out;
}
// Text result
// This code avoids using a QRegExp because QRegExp might be broken.
// Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)
QString sFirstNumber;
while (!remaining.isEmpty() && !remaining.at(0).isSpace()) {
sFirstNumber += remaining.at(0);
remaining.remove(0,1);
}
remaining = remaining.trimmed();
// 4,000 -> 4000
sFirstNumber.remove(',');
// Should now be parseable as floating point
bool ok;
double firstNumber = sFirstNumber.toDouble(&ok);
if (!ok) {
if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";
return out;
}
// Remaining: msec per iteration (total: 4000, iterations: 1)
static const char periterbit[] = " per iteration (total: ";
QString unit;
while (!remaining.startsWith(periterbit) && !remaining.isEmpty()) {
unit += remaining.at(0);
remaining.remove(0,1);
}
if (remaining.isEmpty()) {
if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";
return out;
}
remaining = remaining.mid(sizeof(periterbit)-1);
// Remaining: 4,000, iterations: 1)
static const char itersbit[] = ", iterations: ";
QString sTotal;
while (!remaining.startsWith(itersbit) && !remaining.isEmpty()) {
sTotal += remaining.at(0);
remaining.remove(0,1);
}
if (remaining.isEmpty()) {
if (error) *error = "Could not find pattern: '<number>, iterations: '";
return out;
}
remaining = remaining.mid(sizeof(itersbit)-1);
// 4,000 -> 4000
sTotal.remove(',');
double total = sTotal.toDouble(&ok);
if (!ok) {
if (error) *error = sTotal + " (total) is not a valid number";
return out;
}
// Remaining: 1)
QString sIters;
while (remaining != QLatin1String(")") && !remaining.isEmpty()) {
sIters += remaining.at(0);
remaining.remove(0,1);
}
if (remaining.isEmpty()) {
if (error) *error = "Could not find pattern: '<num>)'";
return out;
}
qint64 iters = sIters.toLongLong(&ok);
if (!ok) {
if (error) *error = sIters + " (iterations) is not a valid integer";
return out;
}
double calcFirstNumber = double(total)/double(iters);
if (!qFuzzyCompare(firstNumber, calcFirstNumber)) {
if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(calcFirstNumber).arg(firstNumber);
return out;
}
out.total = total;
out.unit = unit;
out.iterations = iters;
return out;
}
void tst_Selftests::cleanup()
{
QFETCH(QStringList, loggers);
// Remove the test output files
for (int i = 0; i < loggers.count(); ++i) {
QString logFileName = logName(loggers[i]);
if (!logFileName.isEmpty()) {
QFile logFile(logFileName);
if (logFile.exists())
QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: ").arg(logFileName, logFile.errorString())));
}
}
}
QTEST_MAIN(tst_Selftests)
#include "tst_selftests.moc"
Reference in New Issue View Git Blame Copy Permalink