Add option to compare to different platform in lancelot autotest

(cherry picked from commit b6b160d9ba1e0a5f1fbd67112f8c78ba48e7cad7)

Conflicts:

	tests/baselineserver/src/baselineserver.cpp

Change-Id: Ie8bdff26b8394bb6a5b6b48225b20833dbee0637
Reviewed-on: http://codereview.qt.nokia.com/4082
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Jiang Jiang <jiang.jiang@nokia.com>
This commit is contained in:
aavit 2011-09-01 14:50:59 +02:00 committed by Qt by Nokia
parent 9bb0cc30ee
commit 74cadacdd5
8 changed files with 141 additions and 74 deletions

View File

@ -2306,7 +2306,11 @@ void PaintCommands::command_gradient_setRadialExtended(QRegExp re)
"focal radius=%.2f, spread=%d\n",
cx, cy, rad, fx, fy, frad, m_gradientSpread);
#if QT_VERSION >= 0x040800
QRadialGradient rg(QPointF(cx, cy), rad, QPointF(fx, fy), frad);
#else
QRadialGradient rg(QPointF(cx, cy), rad, QPointF(fx, fy));
#endif
rg.setStops(m_gradientStops);
rg.setSpread(m_gradientSpread);
rg.setCoordinateMode(m_gradientCoordinate);

View File

@ -62,6 +62,7 @@ public:
tst_Lancelot();
static bool simfail;
static PlatformInfo clientInfo;
private:
enum GraphicsEngine {
@ -98,6 +99,7 @@ private slots:
};
bool tst_Lancelot::simfail = false;
PlatformInfo tst_Lancelot::clientInfo;
tst_Lancelot::tst_Lancelot()
{
@ -112,7 +114,7 @@ void tst_Lancelot::initTestCase()
#if defined(Q_OS_SOMEPLATFORM)
QSKIP("This test is not supported on this platform.", SkipAll);
#endif
if (!proto.connect(QLatin1String("tst_Lancelot"), &dryRunMode))
if (!proto.connect(QLatin1String("tst_Lancelot"), &dryRunMode, clientInfo))
QSKIP(qPrintable(proto.errorMessage()), SkipAll);
#if defined(USE_RUNTIME_DIR)
@ -329,13 +331,26 @@ QTEST_MAIN(tst_Lancelot)
int main(int argc, char *argv[])
{
tst_Lancelot::clientInfo = PlatformInfo::localHostInfo();
char *fargv[20];
int fargc = 0;
for (int i = 0; i < qMin(argc, 19); i++) {
if (!qstrcmp(argv[i], "-simfail"))
if (!qstrcmp(argv[i], "-simfail")) {
tst_Lancelot::simfail = true;
else
} else if (!qstrcmp(argv[i], "-compareto") && i < argc-1) {
QString arg = QString::fromLocal8Bit(argv[++i]);
int split = arg.indexOf(QLC('='));
if (split < 0)
continue;
QString key = arg.left(split).trimmed();
QString value = arg.mid(split+1).trimmed();
if (key.isEmpty() || value.isEmpty())
continue;
tst_Lancelot::clientInfo.addOverride(key, value);
} else {
fargv[fargc++] = argv[i];
}
}
fargv[fargc] = 0;
return rmain(fargc, fargv);

View File

@ -83,7 +83,7 @@ void BaselineProtocol::sysSleep(int ms)
}
PlatformInfo::PlatformInfo()
: QMap<QString, QString>(), replaceDefault(false)
: QMap<QString, QString>(), adHoc(true)
{
}
@ -134,11 +134,15 @@ PlatformInfo PlatformInfo::localHostInfo()
pi.insert(PI_GitCommit, QLS("Unknown"));
QByteArray gb = qgetenv("PULSE_GIT_BRANCH");
if (!gb.isEmpty())
if (!gb.isEmpty()) {
pi.insert(PI_PulseGitBranch, QString::fromLatin1(gb));
pi.setAdHocRun(false);
}
QByteArray tb = qgetenv("PULSE_TESTR_BRANCH");
if (!tb.isEmpty())
if (!tb.isEmpty()) {
pi.insert(PI_PulseTestrBranch, QString::fromLatin1(tb));
pi.setAdHocRun(false);
}
return pi;
}
@ -147,43 +151,49 @@ PlatformInfo PlatformInfo::localHostInfo()
PlatformInfo::PlatformInfo(const PlatformInfo &other)
: QMap<QString, QString>(other)
{
sigKeys = other.sigKeys;
replaceDefault = other.replaceDefault;
orides = other.orides;
adHoc = other.adHoc;
}
PlatformInfo &PlatformInfo::operator=(const PlatformInfo &other)
{
QMap<QString, QString>::operator=(other);
sigKeys = other.sigKeys;
replaceDefault = other.replaceDefault;
orides = other.orides;
adHoc = other.adHoc;
return *this;
}
void PlatformInfo::addSignificantKeys(const QStringList &keys, bool replaceDefaultKeys)
void PlatformInfo::addOverride(const QString& key, const QString& value)
{
sigKeys = keys;
replaceDefault = replaceDefaultKeys;
orides.append(key);
orides.append(value);
}
QStringList PlatformInfo::addedKeys() const
QStringList PlatformInfo::overrides() const
{
return sigKeys;
return orides;
}
bool PlatformInfo::addedKeysReplaceDefault() const
void PlatformInfo::setAdHocRun(bool isAdHoc)
{
return replaceDefault;
adHoc = isAdHoc;
}
bool PlatformInfo::isAdHocRun() const
{
return adHoc;
}
QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pi)
{
stream << static_cast<const QMap<QString, QString>&>(pi);
stream << pi.sigKeys << pi.replaceDefault;
stream << pi.orides << pi.adHoc;
return stream;
}
@ -191,7 +201,7 @@ QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pi)
QDataStream & operator>> (QDataStream &stream, PlatformInfo &pi)
{
stream >> static_cast<QMap<QString, QString>&>(pi);
stream >> pi.sigKeys >> pi.replaceDefault;
stream >> pi.orides >> pi.adHoc;
return stream;
}
@ -346,7 +356,7 @@ BaselineProtocol::~BaselineProtocol()
}
bool BaselineProtocol::connect(const QString &testCase, bool *dryrun)
bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo& clientInfo)
{
errMsg.clear();
QByteArray serverName(qgetenv("QT_LANCELOT_SERVER"));
@ -362,7 +372,7 @@ bool BaselineProtocol::connect(const QString &testCase, bool *dryrun)
}
}
PlatformInfo pi = PlatformInfo::localHostInfo();
PlatformInfo pi = clientInfo.isEmpty() ? PlatformInfo::localHostInfo() : clientInfo;
pi.insert(PI_TestCase, testCase);
QByteArray block;
QDataStream ds(&block, QIODevice::ReadWrite);

View File

@ -78,13 +78,14 @@ public:
static PlatformInfo localHostInfo();
void addSignificantKeys(const QStringList& keys, bool replaceDefaultKeys=false);
QStringList addedKeys() const;
bool addedKeysReplaceDefault() const;
void addOverride(const QString& key, const QString& value);
QStringList overrides() const;
bool isAdHocRun() const;
void setAdHocRun(bool isAdHoc);
private:
QStringList sigKeys;
bool replaceDefault;
QStringList orides;
bool adHoc;
friend QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pi);
friend QDataStream & operator>> (QDataStream &stream, PlatformInfo& pi);
};
@ -165,7 +166,7 @@ public:
// For client:
// For advanced client:
bool connect(const QString &testCase, bool *dryrun = 0);
bool connect(const QString &testCase, bool *dryrun = 0, const PlatformInfo& clientInfo = PlatformInfo());
bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
bool submitNewBaseline(const ImageItem &item, QByteArray *serverMsg);
bool submitMismatch(const ImageItem &item, QByteArray *serverMsg);

View File

@ -172,27 +172,31 @@ const char *BaselineHandler::logtime()
bool BaselineHandler::establishConnection()
{
if (!proto.acceptConnection(&plat)) {
if (!proto.acceptConnection(&clientInfo)) {
qWarning() << runId << logtime() << "Accepting new connection from" << proto.socket.peerAddress().toString() << "failed." << proto.errorMessage();
proto.sendBlock(BaselineProtocol::Abort, proto.errorMessage().toLatin1()); // In case the client can hear us, tell it what's wrong.
proto.socket.disconnectFromHost();
return false;
}
QString logMsg;
foreach (QString key, plat.keys()) {
foreach (QString key, clientInfo.keys()) {
if (key != PI_HostName && key != PI_HostAddress)
logMsg += key + QLS(": '") + plat.value(key) + QLS("', ");
logMsg += key + QLS(": '") + clientInfo.value(key) + QLS("', ");
}
qDebug() << runId << logtime() << "Connection established with" << plat.value(PI_HostName)
<< "[" << qPrintable(plat.value(PI_HostAddress)) << "]" << logMsg;
qDebug() << runId << logtime() << "Connection established with" << clientInfo.value(PI_HostName)
<< "[" << qPrintable(clientInfo.value(PI_HostAddress)) << "]" << logMsg
<< "Overrides:" << clientInfo.overrides() << "AdHoc-Run:" << clientInfo.isAdHocRun();
//### Temporarily override the client setting, for client compatibility:
if (!clientInfo.isAdHocRun())
clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty());
settings->beginGroup("ClientFilters");
if (!settings->childKeys().isEmpty() && !plat.value(PI_PulseGitBranch).isEmpty()) { // i.e. not adhoc client
// Abort if client does not match the filters
if (!clientInfo.isAdHocRun()) { // for CI runs, allow filtering of clients. TBD: different filters (settings file) per testCase
foreach (QString filterKey, settings->childKeys()) {
QString filter = settings->value(filterKey).toString();
QString platVal = plat.value(filterKey);
if (filter.isEmpty() || platVal.isEmpty())
QString platVal = clientInfo.value(filterKey);
if (filter.isEmpty())
continue; // tbd: add a syntax for specifying a "value-must-be-present" filter
if (!platVal.contains(filter)) {
qDebug() << runId << logtime() << "Did not pass client filter on" << filterKey << "; disconnecting.";
@ -206,7 +210,7 @@ bool BaselineHandler::establishConnection()
proto.sendBlock(BaselineProtocol::Ack, QByteArray());
report.init(this, runId, plat);
report.init(this, runId, clientInfo);
return true;
}
@ -297,6 +301,12 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline)
ImageItem item;
ds >> item;
if (isBaseline && !clientInfo.overrides().isEmpty()) {
qDebug() << runId << logtime() << "Received baseline from client with override info, ignoring. Item:" << item.itemName;
proto.sendBlock(BaselineProtocol::UnknownError, "New baselines not accepted from client with override info.");
return;
}
QString prefix = pathForItem(item, isBaseline);
qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix;
@ -313,7 +323,7 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline)
cwd.mkpath(dir);
item.image.save(prefix + QLS(FileFormat), FileFormat);
PlatformInfo itemData = plat;
PlatformInfo itemData = clientInfo;
itemData.insert(PI_ImageChecksum, QString::number(item.imageChecksums.at(0), 16)); //# Only the first is stored. TBD: get rid of list
itemData.insert(PI_RunId, runId);
itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString());
@ -366,17 +376,16 @@ void BaselineHandler::receiveDisconnect()
}
void BaselineHandler::mapPlatformInfo() const
PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const
{
mapped = plat;
PlatformInfo mapped = orig;
// Map hostname
QString host = plat.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any
QString host = orig.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any
if (host.isEmpty() || host == QLS("localhost")) {
host = plat.value(PI_HostAddress);
host = orig.value(PI_HostAddress);
} else {
if (!plat.value(PI_PulseGitBranch).isEmpty()) {
// i.e. pulse run, so remove index postfix typical of vm hostnames
if (!orig.isAdHocRun()) { // i.e. CI system run, so remove index postfix typical of vm hostnames
host.remove(QRegExp(QLS("\\d+$")));
if (host.endsWith(QLC('-')))
host.chop(1);
@ -387,18 +396,28 @@ void BaselineHandler::mapPlatformInfo() const
mapped.insert(PI_HostName, host);
// Map qmakespec
QString mkspec = plat.value(PI_QMakeSpec);
QString mkspec = orig.value(PI_QMakeSpec);
mapped.insert(PI_QMakeSpec, mkspec.replace(QLC('/'), QLC('_')));
// Map Qt version
QString ver = plat.value(PI_QtVersion);
QString ver = orig.value(PI_QtVersion);
mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-")));
return mapped;
}
QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) const
{
if (mapped.isEmpty())
mapPlatformInfo();
if (mappedClientInfo.isEmpty()) {
mappedClientInfo = mapPlatformInfo(clientInfo);
PlatformInfo oraw = clientInfo;
for (int i = 0; i < clientInfo.overrides().size()-1; i+=2)
oraw.insert(clientInfo.overrides().at(i), clientInfo.overrides().at(i+1));
overriddenMappedClientInfo = mapPlatformInfo(oraw);
}
const PlatformInfo& mapped = isBaseline ? overriddenMappedClientInfo : mappedClientInfo;
QString itemName = item.itemName.simplified();
itemName.replace(QLC(' '), QLC('_'));
@ -512,13 +531,14 @@ void BaselineHandler::testPathMapping()
item.imageChecksums << 0x0123456789abcdefULL;
item.itemChecksum = 0x0123;
plat.insert(PI_QtVersion, QLS("5.0.0"));
plat.insert(PI_BuildKey, QLS("(nobuildkey)"));
plat.insert(PI_QMakeSpec, QLS("linux-g++"));
plat.insert(PI_PulseGitBranch, QLS("somebranch"));
clientInfo.insert(PI_QtVersion, QLS("5.0.0"));
clientInfo.insert(PI_BuildKey, QLS("(nobuildkey)"));
clientInfo.insert(PI_QMakeSpec, QLS("linux-g++"));
clientInfo.insert(PI_PulseGitBranch, QLS("somebranch"));
foreach(const QString& host, hosts) {
mapped.clear();
plat.insert(PI_HostName, host);
mappedClientInfo.clear();
clientInfo.insert(PI_HostName, host);
qDebug() << "Baseline from" << host << "->" << pathForItem(item, true);
qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false);
}

View File

@ -125,13 +125,14 @@ private:
void storeImage(const QByteArray &itemBlock, bool isBaseline);
void storeItemMetadata(const PlatformInfo &metadata, const QString &path);
PlatformInfo fetchItemMetadata(const QString &path);
void mapPlatformInfo() const;
PlatformInfo mapPlatformInfo(const PlatformInfo& orig) const;
const char *logtime();
QString computeMismatchScore(const QImage& baseline, const QImage& rendered);
BaselineProtocol proto;
PlatformInfo plat;
mutable PlatformInfo mapped;
PlatformInfo clientInfo;
mutable PlatformInfo mappedClientInfo;
mutable PlatformInfo overriddenMappedClientInfo;
QString runId;
bool connectionEstablished;
Report report;

View File

@ -66,12 +66,13 @@ void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo
runId = r;
plat = p;
rootDir = BaselineServer::storagePath() + QLC('/');
reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.value(PI_PulseGitBranch).isEmpty() ? QLS("reports/adhoc/") : QLS("reports/pulse/"));
reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.isAdHocRun() ? QLS("reports/adhoc/") : QLS("reports/pulse/"));
QString dir = rootDir + reportDir;
QDir cwd;
if (!cwd.exists(dir))
cwd.mkpath(dir);
path = reportDir + QLS("Report_") + runId + QLS(".html");
hasOverride = !plat.overrides().isEmpty();
}
void Report::addItems(const ImageItemList &items)
@ -141,11 +142,19 @@ void Report::writeHeader()
<< "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString()
<< " for the test run with id " << runId << "</p>\n"
<< "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</b></span> items reported mismatching</p>\n\n";
out << "<h3>Platform Info:</h3>\n"
out << "<h3>Testing Client Platform Info:</h3>\n"
<< "<table>\n";
foreach (QString key, plat.keys())
out << "<tr><td>" << key << "</td><td>" << plat.value(key) << "</td></tr>\n";
out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n";
out << "</table>\n\n";
if (hasOverride) {
out << "<span style=\"color:red\"><h4>Note! Platform Override Info:</h4></span>\n"
<< "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n"
<< "<table>\n";
for (int i = 0; i < plat.overrides().size()-1; i+=2)
out << "<tr><td>" << plat.overrides().at(i) << ":</td><td>" << plat.overrides().at(i+1) << "</td></tr>\n";
out << "</table>\n\n";
}
}
@ -158,10 +167,12 @@ void Report::writeFunctionResults(const ImageItemList &list)
out << "\n<p>&nbsp;</p><h3>Test function: " << testFunction << "</h3>\n";
out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl
<< "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n";
out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl
<< "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n";
if (!hasOverride) {
out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl
<< "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n";
out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl
<< "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n";
}
out << "<table border=\"2\">\n"
"<tr>\n"
@ -192,10 +203,12 @@ void Report::writeFunctionResults(const ImageItemList &list)
out << "Baseline not found/regenerated";
break;
case ImageItem::IgnoreItem:
out << "<span style=\"background-color:yellow\">Blacklisted</span> "
<< "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx
<< "&itemId=" << item.itemName << "&url=" << pageUrl
<< "\">Whitelist this item</a>";
out << "<span style=\"background-color:yellow\">Blacklisted</span> ";
if (!hasOverride) {
out << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx
<< "&itemId=" << item.itemName << "&url=" << pageUrl
<< "\">Whitelist this item</a>";
}
break;
case ImageItem::Ok:
out << "<span style=\"color:green\"><small>No mismatch reported</small></span>";
@ -224,12 +237,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I
out << "<td align=center>\n"
<< "<p><span style=\"color:red\">Mismatch reported</span></p>\n"
<< "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx
<< "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n"
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx
<< "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered
<< "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n";
if (!hasOverride) {
out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx
<< "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n"
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx
<< "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n";
}
out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered
<< "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n"
<< "</td>\n";
}

View File

@ -86,6 +86,7 @@ private:
int numItems;
int numMismatches;
QTextStream out;
bool hasOverride;
};
#endif // REPORT_H