qt5base-lts/tests/baselineserver/shared/qbaselinetest.cpp
Andrew Knight cd582f8410 Add QT_NO_PROCESS guards in tests where they are missing
A handful of tests lack QT_NO_PROCESS guards, making them non-compilable
on Qt builds with no QProcess support. This commit does not change
QProcess-specific tests, which should be left out of the build using the
.pro file mechanism.

Change-Id: Iac8562428abc1f59ccbb23bf5c3a919785e41f12
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@digia.com>
2013-09-03 08:42:24 +02:00

417 lines
14 KiB
C++

/****************************************************************************
**
** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the test suite of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia. For licensing terms and
** conditions see http://qt.digia.com/licensing. For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights. These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qbaselinetest.h"
#include "baselineprotocol.h"
#include <QtCore/QProcess>
#include <QtCore/QDir>
#define MAXCMDLINEARGS 128
namespace QBaselineTest {
static char *fargv[MAXCMDLINEARGS];
static bool simfail = false;
static PlatformInfo customInfo;
static bool customAutoModeSet = false;
static BaselineProtocol proto;
static bool connected = false;
static bool triedConnecting = false;
static bool dryRunMode = false;
static QByteArray curFunction;
static ImageItemList itemList;
static bool gotBaselines;
static QString definedTestProject;
static QString definedTestCase;
void handleCmdLineArgs(int *argcp, char ***argvp)
{
if (!argcp || !argvp)
return;
bool showHelp = false;
int fargc = 0;
int numArgs = *argcp;
for (int i = 0; i < numArgs; i++) {
QByteArray arg = (*argvp)[i];
QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : 0;
if (arg == "-simfail") {
simfail = true;
} else if (arg == "-auto") {
customAutoModeSet = true;
customInfo.setAdHocRun(false);
} else if (arg == "-adhoc") {
customAutoModeSet = true;
customInfo.setAdHocRun(true);
} else if (arg == "-compareto") {
i++;
int split = qMax(0, nextArg.indexOf('='));
QByteArray key = nextArg.left(split).trimmed();
QByteArray value = nextArg.mid(split+1).trimmed();
if (key.isEmpty() || value.isEmpty()) {
qWarning() << "-compareto requires parameter of the form <key>=<value>";
showHelp = true;
break;
}
customInfo.addOverride(key, value);
} else {
if ( (arg == "-help") || (arg == "--help") )
showHelp = true;
if (fargc >= MAXCMDLINEARGS) {
qWarning() << "Too many command line arguments!";
break;
}
fargv[fargc++] = (*argvp)[i];
}
}
*argcp = fargc;
*argvp = fargv;
if (showHelp) {
// TBD: arrange for this to be printed *after* QTest's help
QTextStream out(stdout);
out << "\n Baseline testing (lancelot) options:\n";
out << " -simfail : Force an image comparison mismatch. For testing purposes.\n";
out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n";
out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n";
out << " -compareto KEY=VAL : Force comparison to baselines from a different client,\n";
out << " for example: -compareto QtVersion=4.8.0\n";
out << " Multiple -compareto client specifications may be given.\n";
out << "\n";
}
}
void addClientProperty(const QString& key, const QString& value)
{
customInfo.insert(key, value);
}
/*
If a client property script is present, run it and accept its output
in the form of one 'key: value' property per line
*/
void fetchCustomClientProperties()
{
#ifdef QT_NO_PROCESS
QSKIP("This test requires QProcess support");
#else
QString script = "hostinfo.sh"; //### TBD: Windows implementation (hostinfo.bat)
QProcess runScript;
runScript.setWorkingDirectory(QCoreApplication::applicationDirPath());
runScript.start("sh", QStringList() << script, QIODevice::ReadOnly);
if (!runScript.waitForFinished(5000) || runScript.error() != QProcess::UnknownError) {
qWarning() << "QBaselineTest: Error running script" << runScript.workingDirectory() + QDir::separator() + script << ":" << runScript.errorString();
qDebug() << " stderr:" << runScript.readAllStandardError().trimmed();
}
while (!runScript.atEnd()) {
QByteArray line = runScript.readLine().trimmed(); // ###local8bit? utf8?
QString key, val;
int colonPos = line.indexOf(':');
if (colonPos > 0) {
key = line.left(colonPos).simplified().replace(' ', '_');
val = line.mid(colonPos+1).trimmed();
}
if (!key.isEmpty() && key.length() < 64 && val.length() < 256) // ###TBD: maximum 256 chars in value?
addClientProperty(key, val);
else
qDebug() << "Unparseable script output ignored:" << line;
}
#endif // !QT_NO_PROCESS
}
bool connect(QByteArray *msg, bool *error)
{
if (connected) {
return true;
}
else if (triedConnecting) {
// Avoid repeated connection attempts, to avoid the program using Timeout * #testItems seconds before giving up
*msg = "Not connected to baseline server.";
*error = true;
return false;
}
triedConnecting = true;
fetchCustomClientProperties();
// Merge the platform info set by the program with the protocols default info
PlatformInfo clientInfo = customInfo;
PlatformInfo defaultInfo = PlatformInfo::localHostInfo();
foreach (QString key, defaultInfo.keys()) {
if (!clientInfo.contains(key))
clientInfo.insert(key, defaultInfo.value(key));
}
if (!customAutoModeSet)
clientInfo.setAdHocRun(defaultInfo.isAdHocRun());
if (!definedTestProject.isEmpty())
clientInfo.insert(PI_Project, definedTestProject);
QString testCase = definedTestCase;
if (testCase.isEmpty() && QTest::testObject() && QTest::testObject()->metaObject()) {
//qDebug() << "Trying to Read TestCaseName from Testlib!";
testCase = QTest::testObject()->metaObject()->className();
}
if (testCase.isEmpty()) {
qWarning("QBaselineTest::connect: No test case name specified, cannot connect.");
return false;
}
if (!proto.connect(testCase, &dryRunMode, clientInfo)) {
*msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
*error = true;
return false;
}
connected = true;
return true;
}
bool disconnectFromBaselineServer()
{
if (proto.disconnect()) {
connected = false;
triedConnecting = false;
return true;
}
return false;
}
bool connectToBaselineServer(QByteArray *msg, const QString &testProject, const QString &testCase)
{
bool dummy;
QByteArray dummyMsg;
definedTestProject = testProject;
definedTestCase = testCase;
return connect(msg ? msg : &dummyMsg, &dummy);
}
void setAutoMode(bool mode)
{
customInfo.setAdHocRun(!mode);
customAutoModeSet = true;
}
void setSimFail(bool fail)
{
simfail = fail;
}
void modifyImage(QImage *img)
{
uint c0 = 0x0000ff00;
uint c1 = 0x0080ff00;
img->setPixel(1,1,c0);
img->setPixel(2,1,c1);
img->setPixel(3,1,c0);
img->setPixel(1,2,c1);
img->setPixel(1,3,c0);
img->setPixel(2,3,c1);
img->setPixel(3,3,c0);
img->setPixel(1,4,c1);
img->setPixel(1,5,c0);
}
bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error)
{
ImageItem item = baseline;
if (simfail) {
// Simulate test failure by forcing image mismatch; for testing purposes
QImage misImg = img;
modifyImage(&misImg);
item.image = misImg;
simfail = false; // One failure is typically enough
} else {
item.image = img;
}
item.imageChecksums.clear();
item.imageChecksums.prepend(ImageItem::computeChecksum(item.image));
QByteArray srvMsg;
switch (baseline.status) {
case ImageItem::Ok:
break;
case ImageItem::IgnoreItem :
qDebug() << msg->constData() << "Ignored, blacklisted on server.";
return true;
break;
case ImageItem::BaselineNotFound:
if (!customInfo.overrides().isEmpty()) {
qWarning() << "Cannot compare to other system's baseline: No such baseline found on server.";
return true;
}
if (proto.submitNewBaseline(item, &srvMsg))
qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded.";
else
qDebug() << msg->constData() << "Baseline not found on server. Uploading of new baseline failed:" << srvMsg;
return true;
break;
default:
qWarning() << "Unexpected reply from baseline server.";
return true;
break;
}
*error = false;
// The actual comparison of the given image with the baseline:
if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) {
if (!proto.submitMatch(item, &srvMsg))
qWarning() << "Failed to report image match to server:" << srvMsg;
return true;
}
bool fuzzyMatch = false;
bool res = proto.submitMismatch(item, &srvMsg, &fuzzyMatch);
if (res && fuzzyMatch) {
*error = true; // To force a QSKIP/debug output; somewhat kludgy
*msg += srvMsg;
return true; // The server decides: a fuzzy match means no mismatch
}
*msg += "Mismatch. See report:\n " + srvMsg;
if (dryRunMode) {
qDebug() << "Dryrun, so ignoring" << *msg;
return true;
}
return false;
}
bool checkImage(const QImage &img, const char *name, quint16 checksum, QByteArray *msg, bool *error, int manualdatatag)
{
if (!connected && !connect(msg, error))
return true;
QByteArray itemName;
bool hasName = qstrlen(name);
const char *tag = QTest::currentDataTag();
if (qstrlen(tag)) {
itemName = tag;
if (hasName)
itemName.append('_').append(name);
} else {
itemName = hasName ? name : "default_name";
}
if (manualdatatag > 0)
{
itemName.prepend("_");
itemName.prepend(QByteArray::number(manualdatatag));
}
*msg = "Baseline check of image '" + itemName + "': ";
ImageItem item;
item.itemName = QString::fromLatin1(itemName);
item.itemChecksum = checksum;
item.testFunction = QString::fromLatin1(QTest::currentTestFunction());
ImageItemList list;
list.append(item);
if (!proto.requestBaselineChecksums(QLatin1String(QTest::currentTestFunction()), &list) || list.isEmpty()) {
*msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
*error = true;
return true;
}
return compareItem(list.at(0), img, msg, error);
}
QTestData &newRow(const char *dataTag, quint16 checksum)
{
if (QTest::currentTestFunction() != curFunction) {
curFunction = QTest::currentTestFunction();
itemList.clear();
gotBaselines = false;
}
ImageItem item;
item.itemName = QString::fromLatin1(dataTag);
item.itemChecksum = checksum;
item.testFunction = QString::fromLatin1(QTest::currentTestFunction());
itemList.append(item);
return QTest::newRow(dataTag);
}
bool testImage(const QImage& img, QByteArray *msg, bool *error)
{
if (!connected && !connect(msg, error))
return true;
if (QTest::currentTestFunction() != curFunction || itemList.isEmpty()) {
qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow()";
return true;
}
if (!gotBaselines) {
if (!proto.requestBaselineChecksums(QString::fromLatin1(QTest::currentTestFunction()), &itemList) || itemList.isEmpty()) {
*msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
*error = true;
return true;
}
gotBaselines = true;
}
QString curTag = QString::fromLatin1(QTest::currentDataTag());
ImageItemList::const_iterator it = itemList.constBegin();
while (it != itemList.constEnd() && it->itemName != curTag)
++it;
if (it == itemList.constEnd()) {
qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow() for row" << curTag;
return true;
}
return compareItem(*it, img, msg, error);
}
}