38be0d1383
This is the beginning of revision history for this module. If you want to look at revision history older than this, please refer to the Qt Git wiki for how to use Git history grafting. At the time of writing, this wiki is located here: http://qt.gitorious.org/qt/pages/GitIntroductionWithQt If you have already performed the grafting and you don't see any history beyond this commit, try running "git log" with the "--follow" argument. Branched from the monolithic repo, Qt master branch, at commit 896db169ea224deb96c59ce8af800d019de63f12
649 lines
17 KiB
C++
649 lines
17 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
|
|
** All rights reserved.
|
|
** Contact: Nokia Corporation (qt-info@nokia.com)
|
|
**
|
|
** This file is part of the test suite of the Qt Toolkit.
|
|
**
|
|
** $QT_BEGIN_LICENSE:LGPL$
|
|
** No Commercial Usage
|
|
** This file contains pre-release code and may not be distributed.
|
|
** You may use this file in accordance with the terms and conditions
|
|
** contained in the Technology Preview License Agreement accompanying
|
|
** this package.
|
|
**
|
|
** GNU Lesser General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU Lesser
|
|
** General Public License version 2.1 as published by the Free Software
|
|
** Foundation and appearing in the file LICENSE.LGPL included in the
|
|
** packaging of this file. Please review the following information to
|
|
** ensure the GNU Lesser General Public License version 2.1 requirements
|
|
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
|
|
**
|
|
** In addition, as a special exception, Nokia gives you certain additional
|
|
** rights. These rights are described in the Nokia Qt LGPL Exception
|
|
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
|
|
**
|
|
** If you have questions regarding the use of this file, please contact
|
|
** Nokia at qt-info@nokia.com.
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
** $QT_END_LICENSE$
|
|
**
|
|
****************************************************************************/
|
|
|
|
|
|
#include <atWrapper.h>
|
|
#include <datagenerator/datagenerator.h>
|
|
|
|
#include <QString>
|
|
#include <QHash>
|
|
#include <QFile>
|
|
#include <QFtp>
|
|
#include <QObject>
|
|
#include <QHostInfo>
|
|
#include <QWidget>
|
|
#include <QImage>
|
|
#include <QtTest/QSignalSpy>
|
|
#include <QLibraryInfo>
|
|
|
|
static const char *ArthurDir = "../../arthur";
|
|
|
|
#include <string.h>
|
|
|
|
atWrapper::atWrapper()
|
|
{
|
|
|
|
// initTests();
|
|
|
|
}
|
|
|
|
bool atWrapper::initTests(bool *haveBaseline)
|
|
{
|
|
qDebug() << "Running test on buildkey:" << QLibraryInfo::buildKey() << " qt version:" << qVersion();
|
|
|
|
qDebug( "Initializing tests..." );
|
|
|
|
if (!loadConfig( QHostInfo::localHostName().split( "." ).first() + ".ini" ))
|
|
return false;
|
|
|
|
//Reset the FTP environment where the results are stored
|
|
*haveBaseline = setupFTP();
|
|
|
|
// Retrieve the latest test result baseline from the FTP server.
|
|
downloadBaseline();
|
|
return true;
|
|
}
|
|
|
|
void atWrapper::downloadBaseline()
|
|
{
|
|
|
|
qDebug() << "Now downloading baseline...";
|
|
|
|
QFtp ftp;
|
|
|
|
QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpMgetAddToList(const QUrlInfo & ) ) );
|
|
|
|
//Making sure that the needed local directories exist.
|
|
|
|
QHashIterator<QString, QString> j(enginesToTest);
|
|
|
|
while ( j.hasNext() )
|
|
{
|
|
j.next();
|
|
|
|
QDir dir( output );
|
|
|
|
if ( !dir.cd( j.key() + ".baseline" ) )
|
|
dir.mkdir( j.key() + ".baseline" );
|
|
|
|
}
|
|
|
|
//FTP to the host specified in the config file, and retrieve the test result baseline.
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.cd( ftpBaseDir );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
while ( i.hasNext() )
|
|
{
|
|
i.next();
|
|
mgetDirList.clear();
|
|
mgetDirList << i.key() + ".baseline";
|
|
ftp.cd( i.key() + ".baseline" );
|
|
ftp.list();
|
|
ftp.cd( ".." );
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
ftpMgetDone( true );
|
|
}
|
|
|
|
ftp.close();
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
}
|
|
|
|
void atWrapper::ftpMgetAddToList( const QUrlInfo &urlInfo )
|
|
{
|
|
//Simply adding to the list of files to download.
|
|
mgetDirList << urlInfo.name();
|
|
|
|
}
|
|
|
|
void atWrapper::ftpMgetDone( bool error)
|
|
{
|
|
Q_UNUSED( error );
|
|
|
|
//Downloading the files listed in mgetDirList...
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
QFile* file;
|
|
|
|
if ( mgetDirList.size() > 1 )
|
|
for ( int i = 1; i < mgetDirList.size(); ++i )
|
|
{
|
|
file = new QFile( QString( output ) + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ) );
|
|
if (file->open(QIODevice::WriteOnly)) {
|
|
ftp.get( ftpBaseDir + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ), file );
|
|
ftp.list(); //Only there to fill up a slot in the pendingCommands queue.
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
file->close();
|
|
} else {
|
|
qDebug() << "Couldn't open file for writing: " << file->fileName();
|
|
}
|
|
}
|
|
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
void atWrapper::uploadFailed( QString dir, QString filename, QByteArray filedata )
|
|
{
|
|
//Upload a failed test case image to the FTP server.
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.cd( ftpBaseDir );
|
|
ftp.cd( dir );
|
|
|
|
ftp.put( filedata, filename, QFtp::Binary );
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
// returns false if no baseline exists
|
|
bool atWrapper::setupFTP()
|
|
{
|
|
qDebug( "Setting up FTP environment" );
|
|
|
|
QString dir = "";
|
|
ftpMkDir( ftpBaseDir );
|
|
|
|
ftpBaseDir += "/" + QLibraryInfo::buildKey();
|
|
|
|
ftpMkDir( ftpBaseDir );
|
|
|
|
ftpBaseDir += "/" + QString( qVersion() );
|
|
|
|
ftpMkDir( ftpBaseDir );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
QHashIterator<QString, QString> j(enginesToTest);
|
|
|
|
bool haveBaseline = true;
|
|
//Creating the baseline directories for each engine
|
|
while ( i.hasNext() )
|
|
{
|
|
i.next();
|
|
//qDebug() << "Creating dir with key:" << i.key();
|
|
ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".failed" );
|
|
ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".diff" );
|
|
if (!ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".baseline" ))
|
|
haveBaseline = false;
|
|
}
|
|
|
|
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.cd( ftpBaseDir );
|
|
//Deleting previous failed directory and all the files in it, then recreating it.
|
|
while ( j.hasNext() )
|
|
{
|
|
j.next();
|
|
rmDirList.clear();
|
|
rmDirList << ftpBaseDir + "/" + j.key() + ".failed" + "/";
|
|
ftpRmDir( j.key() + ".failed" );
|
|
ftp.rmdir( j.key() + ".failed" );
|
|
ftp.mkdir( j.key() + ".failed" );
|
|
ftp.list();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
rmDirList.clear();
|
|
rmDirList << ftpBaseDir + "/" + j.key() + ".diff" + "/";
|
|
ftpRmDir( j.key() + ".diff" );
|
|
ftp.rmdir( j.key() + ".diff" );
|
|
ftp.mkdir( j.key() + ".diff" );
|
|
ftp.list();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
}
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
return haveBaseline;
|
|
}
|
|
|
|
void atWrapper::ftpRmDir( QString dir )
|
|
{
|
|
//Hack to remove a populated directory. (caveat: containing only files and empty dirs, not recursive!)
|
|
qDebug() << "Now removing directory: " << dir;
|
|
QFtp ftp;
|
|
QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpRmDirAddToList(const QUrlInfo & ) ) );
|
|
QObject::connect( &ftp, SIGNAL( done( bool ) ), this, SLOT( ftpRmDirDone( bool ) ) );
|
|
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.list( ftpBaseDir + "/" + dir );
|
|
ftp.close();
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
void atWrapper::ftpRmDirDone( bool error )
|
|
{
|
|
//Deleting each file in the directory listning, rmDirList.
|
|
Q_UNUSED( error );
|
|
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
if ( rmDirList.size() > 1 )
|
|
for (int i = 1; i < rmDirList.size(); ++i)
|
|
ftp.remove( rmDirList.at(0) + rmDirList.at( i ) );
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
// returns false if the directory already exists
|
|
bool atWrapper::ftpMkDir( QString dir )
|
|
{
|
|
//Simply used to avoid QFTP from bailing out and loosing a queue of commands.
|
|
// IE: conveniance.
|
|
QFtp ftp;
|
|
|
|
QSignalSpy commandSpy(&ftp, SIGNAL(commandFinished(int, bool)));
|
|
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
const int command = ftp.mkdir( dir );
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
for (int i = 0; i < commandSpy.count(); ++i)
|
|
if (commandSpy.at(i).at(0) == command)
|
|
return commandSpy.at(i).at(1).toBool();
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
void atWrapper::ftpRmDirAddToList( const QUrlInfo &urlInfo )
|
|
{
|
|
//Just adding the file to the list for deletion
|
|
rmDirList << urlInfo.name();
|
|
}
|
|
|
|
|
|
bool atWrapper::executeTests()
|
|
{
|
|
qDebug("Executing the tests...");
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
|
|
DataGenerator generator;
|
|
|
|
//Running datagenerator against all the frameworks specified in the config file.
|
|
while ( i.hasNext() )
|
|
{
|
|
|
|
i.next();
|
|
|
|
qDebug( "Now testing: " + i.key().toLatin1() );
|
|
|
|
char* params[13];
|
|
//./bin/datagenerator -framework data/framework.ini -engine OpenGL -suite 1.1 -output outtest
|
|
|
|
|
|
QByteArray eng = i.key().toLatin1();
|
|
QByteArray fwk = framework.toLatin1();
|
|
QByteArray sut = suite.toLatin1();
|
|
QByteArray out = output.toLatin1();
|
|
QByteArray siz = size.toLatin1();
|
|
QByteArray fill = fillColor.toLatin1();
|
|
|
|
params[1] = "-framework";
|
|
params[2] = fwk.data();
|
|
params[3] = "-engine";
|
|
params[4] = eng.data();
|
|
params[5] = "-suite";
|
|
params[6] = sut.data();
|
|
params[7] = "-output";
|
|
params[8] = out.data();
|
|
params[9] = "-size";
|
|
params[10] = siz.data();
|
|
params[11] = "-fill";
|
|
params[12] = fill.data();
|
|
|
|
generator.run( 13, params );
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void atWrapper::createBaseline()
|
|
{
|
|
qDebug( "Now uploading a baseline of only the latest test values" );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
|
|
QDir dir( output );
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
ftp.cd( ftpBaseDir );
|
|
//Upload all the latest test results to the FTP server's baseline directory.
|
|
while ( i.hasNext() )
|
|
{
|
|
|
|
i.next();
|
|
dir.cd( i.key() );
|
|
ftp.cd( i.key() + ".baseline" );
|
|
dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
|
|
dir.setNameFilters( QStringList() << "*.png" );
|
|
QFileInfoList list = dir.entryInfoList();
|
|
dir.cd( ".." );
|
|
for (int n = 0; n < list.size(); n++)
|
|
{
|
|
QFileInfo fileInfo = list.at( n );
|
|
QFile file( QString( output ) + "/" + i.key() + "/" + fileInfo.fileName() );
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray fileData = file.readAll();
|
|
//qDebug() << "Sending up:" << fileInfo.fileName() << "with file size" << fileData.size();
|
|
file.close();
|
|
ftp.put( fileData, fileInfo.fileName(), QFtp::Binary );
|
|
}
|
|
|
|
ftp.cd( ".." );
|
|
}
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
bool atWrapper::compare()
|
|
{
|
|
qDebug( "Now comparing the results to the baseline" );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
|
|
while ( i.hasNext() )
|
|
{
|
|
i.next();
|
|
|
|
compareDirs( output , i.key() );
|
|
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void atWrapper::compareDirs( QString basedir, QString target )
|
|
{
|
|
|
|
QDir dir( basedir );
|
|
|
|
/* The following should be redundant now.
|
|
|
|
if ( !dir.cd( target + ".failed" ) )
|
|
dir.mkdir( target + ".failed" );
|
|
else
|
|
dir.cdUp();
|
|
|
|
*/
|
|
|
|
if ( !dir.cd( target + ".diff" ) )
|
|
dir.mkdir( target + ".diff" );
|
|
else
|
|
dir.cdUp();
|
|
|
|
|
|
|
|
//Perform comparisons between the two directories.
|
|
|
|
dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
|
|
dir.setNameFilters( QStringList() << "*.png" );
|
|
dir.cd( target + ".baseline" );
|
|
QFileInfoList list = dir.entryInfoList();
|
|
|
|
for (int i = 0; i < list.size(); ++i)
|
|
{
|
|
QFileInfo fileInfo = list.at(i);
|
|
diff ( basedir, target, fileInfo.fileName() );
|
|
}
|
|
}
|
|
|
|
bool atWrapper::diff( QString basedir, QString dir, QString target )
|
|
{
|
|
//Comparing the two specified files, and then uploading them to
|
|
//the ftp server if they differ
|
|
|
|
basedir += "/" + dir;
|
|
QString one = basedir + ".baseline/" + target;
|
|
QString two = basedir + "/" + target;
|
|
|
|
QFile file( one );
|
|
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray contentsOfOne = file.readAll();
|
|
file.close();
|
|
|
|
file.setFileName( two );
|
|
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray contentsOfTwo = file.readAll();
|
|
file.close();
|
|
|
|
if ( contentsOfTwo.size() == 0 )
|
|
{
|
|
qDebug() << "No test result found for baseline: " << one;
|
|
file.setFileName( one );
|
|
file.open( QIODevice::ReadOnly );
|
|
file.copy( basedir + ".failed/" + target + "_missing" );
|
|
uploadFailed( dir + ".failed", target + "_missing", contentsOfTwo );
|
|
return false;
|
|
}
|
|
|
|
|
|
if ( ( memcmp( contentsOfOne, contentsOfTwo, contentsOfOne.size() ) ) == 0 )
|
|
return true;
|
|
else
|
|
{
|
|
qDebug() << "Sorry, the result did not match: " << one;
|
|
file.setFileName( two );
|
|
file.open( QIODevice::ReadOnly );
|
|
file.copy( basedir + ".failed/" + target );
|
|
file.close();
|
|
uploadFailed( dir + ".failed", target, contentsOfTwo );
|
|
uploadDiff( basedir, dir, target );
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void atWrapper::uploadDiff( QString basedir, QString dir, QString filename )
|
|
{
|
|
|
|
qDebug() << basedir;
|
|
QImage im1( basedir + ".baseline/" + filename );
|
|
QImage im2( basedir + "/" + filename );
|
|
|
|
QImage im3(im1.size(), QImage::Format_ARGB32);
|
|
|
|
im1 = im1.convertToFormat(QImage::Format_ARGB32);
|
|
im2 = im2.convertToFormat(QImage::Format_ARGB32);
|
|
|
|
for ( int y=0; y<im1.height(); ++y )
|
|
{
|
|
uint *s = (uint *) im1.scanLine(y);
|
|
uint *d = (uint *) im2.scanLine(y);
|
|
uint *w = (uint *) im3.scanLine(y);
|
|
|
|
for ( int x=0; x<im1.width(); ++x )
|
|
{
|
|
if (*s != *d)
|
|
*w = 0xff000000;
|
|
else
|
|
*w = 0xffffffff;
|
|
w++;
|
|
s++;
|
|
d++;
|
|
}
|
|
}
|
|
|
|
im3.save( basedir + ".diff/" + filename ,"PNG");
|
|
|
|
QFile file( basedir + ".diff/" + filename );
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray contents = file.readAll();
|
|
file.close();
|
|
|
|
uploadFailed( dir + ".diff", filename, contents );
|
|
|
|
}
|
|
|
|
bool atWrapper::loadConfig( QString path )
|
|
{
|
|
qDebug() << "Loading config file from ... " << path;
|
|
configPath = path;
|
|
//If there is no config file, dont proceed;
|
|
if ( !QFile::exists( path ) )
|
|
{
|
|
return false;
|
|
}
|
|
|
|
|
|
QSettings settings( path, QSettings::IniFormat, this );
|
|
|
|
|
|
//FIXME: Switch to QStringList or something, hash is not needed!
|
|
int numEngines = settings.beginReadArray("engines");
|
|
|
|
for ( int i = 0; i < numEngines; ++i )
|
|
{
|
|
settings.setArrayIndex(i);
|
|
enginesToTest.insert( settings.value( "engine" ).toString(), "Info here please :p" );
|
|
}
|
|
|
|
settings.endArray();
|
|
|
|
framework = QString(ArthurDir) + QDir::separator() + settings.value( "framework" ).toString();
|
|
suite = settings.value( "suite" ).toString();
|
|
output = settings.value( "output" ).toString();
|
|
size = settings.value( "size", "480,360" ).toString();
|
|
fillColor = settings.value( "fill", "white" ).toString();
|
|
ftpUser = settings.value( "ftpUser" ).toString();
|
|
ftpPass = settings.value( "ftpPass" ).toString();
|
|
ftpHost = settings.value( "ftpHost" ).toString();
|
|
ftpBaseDir = settings.value( "ftpBaseDir" ).toString();
|
|
|
|
|
|
QDir::current().mkdir( output );
|
|
|
|
output += "/" + QLibraryInfo::buildKey();
|
|
|
|
QDir::current().mkdir( output );
|
|
|
|
output += "/" + QString( qVersion() );
|
|
|
|
QDir::current().mkdir( output );
|
|
|
|
|
|
ftpBaseDir += "/" + QHostInfo::localHostName().split( "." ).first();
|
|
|
|
|
|
/*
|
|
framework = "data/framework.ini";
|
|
suite = "1.1";
|
|
output = "testresults";
|
|
ftpUser = "anonymous";
|
|
ftpPass = "anonymouspass";
|
|
ftpHost = "kramer.troll.no";
|
|
ftpBaseDir = "/arthurtest";
|
|
*/
|
|
return true;
|
|
}
|
|
|
|
bool atWrapper::runAutoTests()
|
|
{
|
|
//SVG needs this widget...
|
|
QWidget dummy;
|
|
|
|
bool haveBaseline = false;
|
|
|
|
if (!initTests(&haveBaseline))
|
|
return false;
|
|
executeTests();
|
|
|
|
if ( !haveBaseline )
|
|
{
|
|
qDebug( " First run! Creating baseline..." );
|
|
createBaseline();
|
|
}
|
|
else
|
|
{
|
|
qDebug( " Comparing results..." );
|
|
compare();
|
|
}
|
|
return true;
|
|
}
|